From 357c3600b1bf22e94efe9fd09f311f4b5c485143 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 5 Nov 2026 00:54:39 +0000 Subject: [PATCH 01/53] feat: keep rollouts through a lost interception tunnel Add an in-box relay (`interception/relay`) that a harness in a remote box reaches the interception server through, at 127.0.0.1, instead of the tunnel URL (`agent.relay`, off by default). The relay retries a model call for up to 5 minutes while the tunnel cannot be reached, and marks each retry so the interception server joins it to the call it already serves. When a tunnel is lost for good, its server still takes no new rollouts, but no longer fails the ones that reach it through a relay: it opens a new tunnel, allows its host in each box's egress policy and reroutes each relay. Rollouts with tool servers or deployment services still fail, since those reach the old URL. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 9 + tests/integration/test_relay.py | 201 +++++++ tests/unit/interception/test_relay.py | 441 ++++++++++++++ tests/unit/interception/test_tunnel.py | 77 ++- verifiers/v1/configs/agent.py | 7 + verifiers/v1/interception/relay/__init__.py | 139 +++++ verifiers/v1/interception/relay/server.py | 622 ++++++++++++++++++++ verifiers/v1/interception/server.py | 65 +- verifiers/v1/rollout.py | 56 +- verifiers/v1/session.py | 4 + 10 files changed, 1607 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_relay.py create mode 100644 tests/unit/interception/test_relay.py create mode 100644 verifiers/v1/interception/relay/__init__.py create mode 100644 verifiers/v1/interception/relay/server.py diff --git a/docs/architecture.md b/docs/architecture.md index 8e9057e599..c1e033eb6e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -25,6 +25,15 @@ Routing traffic through the interception server enables several things beyond fo - Applying sampling parameters in harnesses that don't expose those settings directly - Running the task's [stop and interception hooks](tasksets.md#stops-and-interception) to inspect or rewrite tool responses and server-side web search results (for example, to block reward hacks) +### Relay + +A Prime tunnel can drop for a while (its client reconnects) or for good (the tunnel service drops its registration). A lost tunnel fails every rollout on its interception server. With `[env.agent] relay = true`, a harness in a remote box reaches the interception server through a relay in its own box, at `127.0.0.1`, instead of the tunnel URL: + +- While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. +- When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. + +A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay needs Python 3.10 or later in the box. + ## Finishing a rollout When the agent finishes its turn normally, the rollout collects artifacts and runs in-place grading before closing the harness session: diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py new file mode 100644 index 0000000000..535eceda85 --- /dev/null +++ b/tests/integration/test_relay.py @@ -0,0 +1,201 @@ +"""The interception relay on Prime: the server's tunnel is lost while a rollout's model +call is in flight: its registration is deleted and its frpc connection drops, so it +cannot log in again. With `agent.relay`, the server opens a new tunnel, reroutes the box's +relay to it, and the rollout finishes as if nothing happened: the waiting model call +goes through once, on one branch. Without the relay, the rollout fails, as before. + +The model is scripted and holds its second reply until the test has dropped the tunnel +and, with the relay, until the server has replaced it.""" + +import asyncio +import json +import os +import signal +from typing import Self + +import pytest +from aiohttp import web + +import verifiers.v1 as vf +from verifiers.v1.dialects.chat import ChatDialect +from verifiers.v1.harnesses.bash import BashHarnessConfig +from verifiers.v1.interception import InterceptionServer +from verifiers.v1.utils.prime import load_prime_config + +pytestmark = pytest.mark.skipif( + not (os.environ.get("PRIME_API_KEY") or load_prime_config().get("api_key")), + reason="needs Prime credentials", +) + +DETECTION_SECONDS = 180 +"""How long the tunnel's health probe may take to declare the lost tunnel gone, plus the +time to open a new one.""" + + +def _tool_call(command: str, n: int) -> dict: + call = {"name": "bash", "arguments": json.dumps({"command": command})} + return { + "role": "assistant", + "content": "", + "tool_calls": [{"id": f"call_{n}", "type": "function", "function": call}], + } + + +SCRIPT = [ + _tool_call("echo before-the-tunnel-loss > /tmp/note", 0), + _tool_call("cat /tmp/note", 1), + {"role": "assistant", "content": "done"}, +] + + +class HeldModel: + """A chat-completions model that plays `SCRIPT`, one reply per turn, and holds the + reply to turn 1 until `release` is set. It counts the requests per turn.""" + + def __init__(self) -> None: + self.requests: dict[int, int] = {} + self.held = asyncio.Event() + self.release = asyncio.Event() + self.url = "" + self.runner: web.AppRunner | None = None + + async def answer(self, request: web.Request) -> web.StreamResponse: + body = await request.json() + turn = sum(1 for m in body["messages"] if m["role"] == "assistant") + self.requests[turn] = self.requests.get(turn, 0) + 1 + if turn == 1: + self.held.set() + await self.release.wait() + message = SCRIPT[min(turn, len(SCRIPT) - 1)] + raw = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 0, + "model": body.get("model", "scripted"), + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" + if "tool_calls" in message + else "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + dialect = ChatDialect() + if not dialect.streaming(body): + return web.json_response(raw) + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + for event in dialect.stream_events(raw): + await response.write(event) + await response.write_eof() + return response + + async def __aenter__(self) -> Self: + app = web.Application() + app.router.add_post("/{tail:.*}", self.answer) + self.runner = web.AppRunner(app) + await self.runner.setup() + site = web.TCPSite(self.runner, "127.0.0.1", 0) + await site.start() + self.url = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1" + return self + + async def __aexit__(self, *exc) -> None: + self.release.set() + assert self.runner is not None + await self.runner.cleanup() + + +async def drop_tunnel(url: str) -> None: + """Lose the tunnel that serves `url` for good: delete its registration, then end its + frpc connection, which then cannot log in again. A deleted registration alone keeps + serving while frpc stays connected.""" + from prime_tunnel import TunnelClient + + async with TunnelClient() as api: + tunnels = await api.list_tunnels(per_page=100) + (tunnel,) = [t for t in tunnels if t.url.rstrip("/") == url.rstrip("/")] + assert await api.delete_tunnel(tunnel.tunnel_id) + frpc = [pid for pid in _children() if f"{tunnel.tunnel_id}.toml" in _cmdline(pid)] + assert frpc, "no frpc process for the tunnel" + for pid in frpc: + os.kill(pid, signal.SIGKILL) + + +def _children() -> list[int]: + pids = [] + for entry in os.listdir("/proc"): + if not entry.isdigit(): + continue + try: + with open(f"/proc/{entry}/stat") as stat: + ppid = int(stat.read().rsplit(")", 1)[1].split()[1]) + except (OSError, ValueError, IndexError): + continue + if ppid == os.getpid(): + pids.append(int(entry)) + return pids + + +def _cmdline(pid: int) -> str: + try: + with open(f"/proc/{pid}/cmdline") as cmdline: + return cmdline.read() + except OSError: + return "" + + +async def _run(relay: bool) -> tuple[vf.Trace, HeldModel, str, str]: + async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: + agent = vf.make_agent( + vf.AgentConfig( + harness=BashHarnessConfig(id="bash"), + runtime=vf.PrimeConfig(image="python:3.11-slim", labels=["vf-ci"]), + model="scripted", + client=vf.EvalClientConfig(base_url=model.url), + relay=relay, + ), + interception=server, + ) + run = asyncio.create_task(agent.run(vf.Task(vf.TaskData(idx=0, prompt="Go.")))) + await asyncio.wait_for(model.held.wait(), 600) + first_url = server.base_url + await drop_tunnel(first_url) + if relay: + # Hold the reply until the server serves the rollout through a new tunnel. + async with asyncio.timeout(DETECTION_SECONDS): + while server.base_url == first_url: + await asyncio.sleep(1) + else: + async with asyncio.timeout(DETECTION_SECONDS): + while server.healthy: + await asyncio.sleep(1) + model.release.set() + trace = await asyncio.wait_for(run, 600) + return trace, model, first_url, server.base_url + + +async def test_rollout_survives_a_replaced_tunnel(): + trace, model, first_url, last_url = await _run(relay=True) + + assert last_url != first_url + assert trace.ok, [error.message for error in trace.errors] + assert trace.errors == [] + tool_results = [str(m.content) for m in trace.messages if m.role == "tool"] + assert tool_results[-1].strip() == "before-the-tunnel-loss" + # The held call reached the model once: the relay's retry joined the request the + # server was already serving. + assert model.requests[1] == 1 + assert len(trace.branches) == 1 + + +async def test_rollout_without_relay_fails_on_a_lost_tunnel(): + trace, _, _, _ = await _run(relay=False) + + assert not trace.ok + # The harness gets the tunnel edge's 404 and exits, usually before the server + # declares the tunnel gone and fails the rollout itself. + assert {error.type for error in trace.errors} & {"HarnessError", "TunnelError"} diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py new file mode 100644 index 0000000000..3dd0b47a6f --- /dev/null +++ b/tests/unit/interception/test_relay.py @@ -0,0 +1,441 @@ +"""The in-box relay forwards requests to its upstream with swapped tokens, streams +responses as they arrive, and on SIGHUP moves requests still waiting for their first +event to the new upstream on the same client connection.""" + +import http.client +import json +import os +import queue +import signal +import socket +import subprocess +import sys +import threading +import time +from collections.abc import Callable, Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from verifiers.v1.interception.relay import RELAY_SOURCE, relay_config + +TIMEOUT = 5 + + +class FakeUpstream: + """An HTTP server whose handler each test sets; it records every request.""" + + def __init__(self, port: int = 0) -> None: + self.requests: queue.Queue[dict] = queue.Queue() + self.respond: Callable[[BaseHTTPRequestHandler], None] = lambda h: None + self.stop = threading.Event() + upstream = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format, *args) -> None: + pass + + def handle_request(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + upstream.requests.put( + { + "method": self.command, + "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": self.rfile.read(length), + } + ) + try: + upstream.respond(self) + except OSError: + self.close_connection = True + + do_GET = do_POST = handle_request + + self.server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + self.server.daemon_threads = True + # The relay resets connections whose response it abandons. + self.server.handle_error = lambda request, address: None + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread( + target=self.server.serve_forever, args=(0.05,), daemon=True + ).start() + + def close(self) -> None: + self.stop.set() + self.server.shutdown() + self.server.server_close() + + +def reply(handler, status: int, body: dict) -> None: + data = json.dumps(body).encode() + handler.send_response(status) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def start_sse(handler) -> None: + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + + +def chunk(handler, data: bytes) -> None: + handler.wfile.write(b"%x\r\n%s\r\n" % (len(data), data)) + handler.wfile.flush() + + +def end_sse(handler) -> None: + handler.wfile.write(b"0\r\n\r\n") + handler.wfile.flush() + + +def keepalives(upstream: FakeUpstream, before: bytes = b""): + """An SSE response that sends `before`, then only keepalives until it is cut.""" + + def respond(handler) -> None: + start_sse(handler) + if before: + chunk(handler, before) + while not upstream.stop.is_set(): + chunk(handler, b": keepalive\n") + upstream.stop.wait(0.05) + + return respond + + +def events(*data: bytes): + def respond(handler) -> None: + start_sse(handler) + for event in data: + chunk(handler, event) + end_sse(handler) + + return respond + + +class Relay: + def __init__( + self, + tmp_path: Path, + upstream: str, + tokens: dict[str, str], + source: str, + retry_seconds: float = 300.0, + ) -> None: + self.retry_seconds = retry_seconds + script = tmp_path / "relay_server.py" + script.write_text(source) + self.config = tmp_path / "config.json" + self.write(upstream, tokens) + ready = tmp_path / "port" + self.stderr = (tmp_path / "relay.log").open("w+") + self.process = subprocess.Popen( + [sys.executable, str(script), "--config", str(self.config)] + + ["--port", "0", "--ready", str(ready)], + stderr=self.stderr, + ) + self.port = int(wait_for(lambda: ready.exists() and ready.read_text())) + + def write(self, upstream: str, tokens: dict[str, str]) -> None: + self.config.write_bytes( + relay_config(upstream, tokens, retry_seconds=self.retry_seconds) + ) + + def reload(self, upstream: str, tokens: dict[str, str]) -> None: + self.write(upstream, tokens) + os.kill(self.process.pid, signal.SIGHUP) + + def connect(self) -> http.client.HTTPConnection: + return http.client.HTTPConnection("127.0.0.1", self.port, timeout=TIMEOUT) + + def close(self) -> str: + self.process.kill() + self.process.wait() + self.stderr.seek(0) + return self.stderr.read() + + +def wait_for(condition: Callable, timeout: float = TIMEOUT): + deadline = time.monotonic() + timeout + while not (value := condition()): + if time.monotonic() > deadline: + raise TimeoutError("condition not met") + time.sleep(0.01) + return value + + +def read_until(response: http.client.HTTPResponse, marker: bytes) -> bytes: + data = b"" + while marker not in data: + piece = response.read1(65536) + if not piece: + raise EOFError(data) + data += piece + return data + + +@pytest.fixture +def upstreams() -> Iterator[tuple[FakeUpstream, FakeUpstream]]: + a, b = FakeUpstream(), FakeUpstream() + yield a, b + a.close() + b.close() + + +@pytest.fixture +def make_relay(tmp_path) -> Iterator[Callable[..., Relay]]: + relays: list[Relay] = [] + + def make( + upstream: str, + tokens: dict[str, str] | None = None, + source: str = RELAY_SOURCE, + retry_seconds: float = 300.0, + ) -> Relay: + relays.append(Relay(tmp_path, upstream, tokens or {}, source, retry_seconds)) + return relays[-1] + + yield make + for relay in relays: + sys.stderr.write(relay.close()) + + +def sse_post(relay: Relay, token: str = "box") -> http.client.HTTPResponse: + connection = relay.connect() + connection.request( + "POST", + "/v1/chat/completions", + b'{"stream": true}', + {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + ) + return connection.getresponse() + + +def test_forwards_with_swapped_tokens(upstreams, make_relay): + upstream, _ = upstreams + upstream.respond = lambda h: reply(h, 401, {"error": "nope"}) + relay = make_relay(f"{upstream.url}/base/", {"box": "real", "key": "real-key"}) + connection = relay.connect() + for headers, expected in ( + ({"Authorization": "Bearer box"}, ("Bearer real", None)), + ({"x-api-key": "key"}, (None, "real-key")), + ( + {"Authorization": "Bearer other", "x-api-key": "other"}, + ("Bearer other", "other"), + ), + ): + connection.request("POST", "/v1/messages?beta=true", b'{"a": 1}', headers) + response = connection.getresponse() + assert response.status == 401 + assert response.getheader("Content-Type") == "application/json" + assert json.loads(response.read()) == {"error": "nope"} + seen = upstream.requests.get(timeout=TIMEOUT) + assert seen["method"] == "POST" + assert seen["path"] == "/base/v1/messages?beta=true" + assert seen["body"] == b'{"a": 1}' + assert seen["headers"]["host"] == upstream.url.removeprefix("http://") + auth = (seen["headers"].get("authorization"), seen["headers"].get("x-api-key")) + assert auth == expected + + +def test_streams_sse_as_it_arrives(upstreams, make_relay): + upstream, _ = upstreams + received = threading.Event() + + def respond(handler) -> None: + start_sse(handler) + chunk(handler, b"data: 1\n\n") + received.wait(TIMEOUT) + chunk(handler, b"data: 2\n\n") + end_sse(handler) + + upstream.respond = respond + response = sse_post(make_relay(upstream.url)) + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert read_until(response, b"data: 1\n\n") == b"data: 1\n\n" + received.set() + assert response.read() == b"data: 2\n\n" + + +def test_sighup_retries_waiting_request_on_new_upstream(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a) + b.respond = events(b"data: from-b\n\n", b"data: [DONE]\n\n") + relay = make_relay(a.url, {"box": "secret-a"}) + response = sse_post(relay) + assert response.status == 200 + read_until(response, b": keepalive\n") + assert ( + a.requests.get(timeout=TIMEOUT)["headers"]["authorization"] == "Bearer secret-a" + ) + relay.reload(b.url, {"box": "secret-b"}) + body = response.read() + assert body.endswith(b"data: from-b\n\ndata: [DONE]\n\n") + assert b"data:" not in body.partition(b"data: from-b")[0] + seen = b.requests.get(timeout=TIMEOUT) + assert seen["headers"]["authorization"] == "Bearer secret-b" + assert seen["headers"]["x-stainless-retry-count"] == "1" + assert seen["body"] == b'{"stream": true}' + + +def test_sighup_keeps_client_alive_until_new_upstream_is_up(upstreams, make_relay): + a, _ = upstreams + a.respond = keepalives(a) + with socket.socket() as free: + free.bind(("127.0.0.1", 0)) + port = free.getsockname()[1] + source = RELAY_SOURCE.replace( + "KEEPALIVE_SECONDS = 10.0", "KEEPALIVE_SECONDS = 0.05" + ) + relay = make_relay(a.url, source=source) + response = sse_post(relay) + read_until(response, b": keepalive\n") + relay.reload(f"http://127.0.0.1:{port}", {}) + read_until(response, b": relay keepalive\n") + b = FakeUpstream(port) + try: + b.respond = events(b"data: from-b\n\n") + assert response.read().endswith(b"data: from-b\n\n") + finally: + b.close() + + +def test_sighup_retry_with_other_status_sends_error_event(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a) + b.respond = lambda h: reply(h, 500, {"error": "boom"}) + relay = make_relay(a.url) + response = sse_post(relay) + read_until(response, b": keepalive\n") + relay.reload(b.url, {}) + assert response.read().endswith( + b'event: error\ndata: {"error": "relay: upstream changed status on retry"}\n\n' + ) + + +def test_sighup_after_data_closes_client_connection(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a, before=b"data: partial\n\n") + relay = make_relay(a.url) + response = sse_post(relay) + read_until(response, b"data: partial\n\n") + relay.reload(b.url, {}) + with pytest.raises((http.client.HTTPException, OSError)): + while response.read1(65536): + pass + assert b.requests.empty() + + +def test_concurrent_requests_do_not_block_each_other(upstreams, make_relay): + upstream, _ = upstreams + both = threading.Barrier(2, timeout=TIMEOUT) + + def respond(handler) -> None: + both.wait() + events(b"data: done\n\n")(handler) + + upstream.respond = respond + relay = make_relay(upstream.url) + bodies: list[bytes] = [] + + def call() -> None: + bodies.append(sse_post(relay).read()) + + threads = [threading.Thread(target=call) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(TIMEOUT) + assert bodies == [b"data: done\n\n"] * 2 + + +def test_unreachable_upstream_answers_502_after_the_retry_window(make_relay): + relay = make_relay("http://127.0.0.1:1", retry_seconds=1) + started = time.monotonic() + response = sse_post(relay) + assert response.status == 502 + assert "unreachable for 1s" in json.loads(response.read())["error"] + assert 1 <= time.monotonic() - started < 5 + + +def edge_page(status: int): + """A tunnel edge's own error page.""" + + def respond(handler) -> None: + body = b"Tunnel Not Found" + handler.send_response(status) + handler.send_header("Content-Type", "text/html") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + return respond + + +def test_edge_error_pages_are_retried(upstreams, make_relay): + a, _ = upstreams + answers = [edge_page(404), edge_page(502)] + + def respond(handler) -> None: + if answers: + answers.pop(0)(handler) + else: + reply(handler, 200, {"ok": True}) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request( + "POST", "/v1/chat/completions", b"{}", {"Authorization": "Bearer box"} + ) + response = connection.getresponse() + assert response.status == 200 + assert json.loads(response.read()) == {"ok": True} + retries = [ + a.requests.get(timeout=TIMEOUT)["headers"].get("x-stainless-retry-count") + for _ in range(3) + ] + # Each retry is marked, so the interception server joins it to the request it serves. + assert retries == [None, "1", "2"] + + +def test_interception_errors_reach_the_harness(upstreams, make_relay): + a, _ = upstreams + a.respond = lambda handler: reply(handler, 503, {"error": "provider down"}) + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + response = connection.getresponse() + assert response.status == 503 + assert json.loads(response.read()) == {"error": "provider down"} + assert a.requests.qsize() == 1 + + +def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): + """The host replaces a lost tunnel: the request waits on the dead URL and goes + through once the relay is rerouted to the new one.""" + _, b = upstreams + b.respond = events(b"data: from-b\n\n") + relay = make_relay("http://127.0.0.1:1") + connection = relay.connect() + connection.request( + "POST", + "/v1/chat/completions", + b'{"stream": true}', + {"Authorization": "Bearer box"}, + ) + time.sleep(1) # the request is retrying against the dead upstream + relay.reload(b.url, {}) + response = connection.getresponse() + assert response.status == 200 + assert response.read().endswith(b"data: from-b\n\n") + retried = b.requests.get(timeout=TIMEOUT)["headers"]["x-stainless-retry-count"] + assert int(retried) >= 1 diff --git a/tests/unit/interception/test_tunnel.py b/tests/unit/interception/test_tunnel.py index 4c2ccc0f0f..60b60619b9 100644 --- a/tests/unit/interception/test_tunnel.py +++ b/tests/unit/interception/test_tunnel.py @@ -1,5 +1,6 @@ """A prime tunnel is kept serving by the SDK through the service's health route. A tunnel -that can never work again retires its server: its rollouts fail and the pools skip it.""" +that can never work again retires its server: the pools skip it, its rollouts that reach +it through a relay move to a new tunnel, and the others fail.""" import asyncio import logging @@ -98,7 +99,9 @@ async def test_server_answers_the_health_route_on_a_connection_it_closes(): def test_gone_tunnel_fails_the_servers_rollouts_and_refuses_new_ones(): server = InterceptionServer() released = [] - session = SimpleNamespace(error=None, release=lambda: released.append(True)) + session = SimpleNamespace( + error=None, release=lambda: released.append(True), reroute=None + ) server.sessions["secret"] = session server._handle_tunnel_gone() @@ -109,6 +112,72 @@ def test_gone_tunnel_fails_the_servers_rollouts_and_refuses_new_ones(): server.register(SimpleNamespace()) +class ReplacingTunnel: + """Exposes the server at a new URL each time; `fail` makes the next expose fail.""" + + def __init__(self) -> None: + self.on_gone = None + self.urls = iter(["https://first.example", "https://second.example"]) + self.fail = False + + @asynccontextmanager + async def expose(self, port): + if self.fail: + raise TunnelError("no tunnel") + yield next(self.urls) + + +def movable_session(trace_id: str, moved: list) -> SimpleNamespace: + async def reroute(url: str) -> None: + moved.append((trace_id, url)) + + session = SimpleNamespace( + error=None, released=False, reroute=reroute, trace=SimpleNamespace(id=trace_id) + ) + session.release = lambda: setattr(session, "released", True) + return session + + +async def test_gone_tunnel_moves_movable_rollouts_to_a_new_tunnel(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.base_url = await server.stack.enter_async_context(server.tunnel.expose(0)) + moved: list = [] + movable = movable_session("movable", moved) + stuck = SimpleNamespace( + error=None, released=False, reroute=None, trace=SimpleNamespace(id="stuck") + ) + stuck.release = lambda: setattr(stuck, "released", True) + server.sessions.update(movable=movable, stuck=stuck) + + server._handle_tunnel_gone() + await server._replacing + + assert moved == [("movable", "https://second.example")] + assert server.base_url == "https://second.example" + assert movable.error is None and not movable.released + assert isinstance(stuck.error, TunnelError) and stuck.released + # The server still takes no new rollouts; the pools retire it once these leave. + assert not server.healthy + await server.stop() + + +async def test_failed_replacement_fails_the_movable_rollouts(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.tunnel.fail = True + moved: list = [] + movable = movable_session("movable", moved) + server.sessions["movable"] = movable + + server._handle_tunnel_gone() + await server._replacing + + assert moved == [] + assert isinstance(movable.error, TunnelError) and movable.released + await server.stop() + + async def test_elastic_pool_replaces_a_retired_server_and_frees_it(): async with ElasticInterceptionPool() as pool: retired = await pool._warm_task @@ -127,7 +196,9 @@ async def test_elastic_pool_replaces_a_retired_server_and_frees_it(): async def test_elastic_pool_keeps_a_retired_server_until_its_rollouts_leave(): async with ElasticInterceptionPool() as pool: retired = await pool._warm_task - retired.sessions["secret"] = SimpleNamespace(error=None, release=lambda: None) + retired.sessions["secret"] = SimpleNamespace( + error=None, release=lambda: None, reroute=None + ) retired._handle_tunnel_gone() replacement = await pool._server() diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 19d484563d..7691863fe4 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -79,6 +79,13 @@ class AgentConfig(BaseConfig): network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" + relay: bool = False + """Reach the interception server through a relay in a remote box (127.0.0.1) instead + of its tunnel URL. The relay retries a model call while the tunnel reconnects, and + when a tunnel is lost for good, the server opens a new one and reroutes the relay, + so the rollout goes on. A rollout with tool servers or deployment services still + fails on a lost tunnel: they reach the old URL. Needs Python 3.10+ in the box.""" + @field_serializer("scoring") def _scoring_overrides( self, scoring: ScoringConfig, info: SerializationInfo diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py new file mode 100644 index 0000000000..8e87e84dfb --- /dev/null +++ b/verifiers/v1/interception/relay/__init__.py @@ -0,0 +1,139 @@ +"""Host side of the in-box relay (`server.py`). + +A harness in a remote box can reach the interception server through a loopback relay in +the box, at `RELAY_URL`, with the box tokens of the relay config as its secrets. The relay +forwards to the current upstream and swaps each box token for its upstream token. When +the upstream URL changes (the server replaced a lost tunnel), the host points it at the +new one: + +1. Write `relay_config(upstream, tokens)` to `RELAY_CONFIG_PATH`, creating `RELAY_DIR`. +2. Start `[*await runtime.prepare_uv_script(RELAY_SOURCE, activate=False), + *relay_args()]` in the background, and wait for `RELAY_READY_PATH` to exist. It + holds the listening port; `RELAY_PID_PATH` holds the process id. +3. To reroute, rewrite `RELAY_CONFIG_PATH` and run `RELAY_RELOAD_COMMAND`. The relay + re-reads the config, aborts every in-flight upstream request and retries each one + that has sent its client only the status, headers and SSE comments. A request that + already streamed data downstream has its client connection closed instead. +""" + +from __future__ import annotations + +import asyncio +import shlex +import time +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, field_validator + +from verifiers.v1.errors import SandboxError + +if TYPE_CHECKING: + from verifiers.v1.runtimes.runtime import Runtime + +RELAY_SOURCE = (Path(__file__).resolve().parent / "server.py").read_text() +RELAY_DIR = "/tmp/vf-relay" +RELAY_CONFIG_PATH = f"{RELAY_DIR}/config.json" +"""The relay's config: `RelayConfig` as JSON. It holds upstream secrets.""" +RELAY_READY_PATH = f"{RELAY_DIR}/port" +"""Written with the listening port (decimal, newline-terminated) once the relay accepts +connections.""" +RELAY_PID_PATH = f"{RELAY_DIR}/pid" +"""Written with the relay's process id before `RELAY_READY_PATH`.""" +RELAY_PORT = 47821 +"""A fixed port, so the harness URL is known before the relay starts. Port 0 picks a +free one instead, read back from `RELAY_READY_PATH`.""" +RELAY_URL = f"http://127.0.0.1:{RELAY_PORT}" +RELAY_RELOAD_COMMAND = ["sh", "-c", f'kill -HUP "$(cat {RELAY_PID_PATH})"'] +"""Make the relay re-read `RELAY_CONFIG_PATH` (SIGHUP). An invalid config is logged and +ignored, keeping the old upstream.""" + + +class RelayConfig(BaseModel): + """The relay's config file.""" + + model_config = ConfigDict(extra="forbid") + + upstream: str + """Base URL that request paths are appended to; it may carry a path prefix.""" + tokens: dict[str, str] = {} + """Box token to upstream token, swapped in `Authorization: Bearer` and `x-api-key`.""" + retry_seconds: float = 300.0 + """How long the relay retries a request while the upstream cannot be reached.""" + + @field_validator("upstream") + @classmethod + def _http_url(cls, upstream: str) -> str: + parts = urlsplit(upstream) + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError(f"upstream must be an http(s) URL: {upstream!r}") + if parts.query or parts.fragment: + raise ValueError(f"upstream must not have a query: {upstream!r}") + return upstream + + +def relay_config(upstream: str, tokens: dict[str, str], **options: Any) -> bytes: + """The contents of `RELAY_CONFIG_PATH`; `options` sets the other `RelayConfig` fields.""" + return ( + RelayConfig(upstream=upstream, tokens=tokens, **options) + .model_dump_json() + .encode() + ) + + +def relay_args( + port: int = RELAY_PORT, + config: str = RELAY_CONFIG_PATH, + ready: str = RELAY_READY_PATH, + pid: str = RELAY_PID_PATH, +) -> list[str]: + """The relay's arguments, after the argv of its prepared script.""" + return ["--config", config, "--port", str(port), "--ready", ready, "--pid", pid] + + +RELAY_START_SECONDS = 60 +"""How long a started relay gets to accept connections.""" + + +async def start( + runtime: Runtime, name: str, upstream: str, tokens: dict[str, str] +) -> dict[str, Any]: + """Start the relay in `runtime`'s box, forwarding to `upstream` with `tokens`, and + return what a later process needs to reroute it (`reroute`). `name` labels its log.""" + await _write_config(runtime, upstream, tokens) + program = await runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + await runtime.run_background( + [*program, *relay_args()], {}, log=f"{RELAY_DIR}/{name}.log" + ) + deadline = time.monotonic() + RELAY_START_SECONDS + while (await runtime.run(["test", "-s", RELAY_READY_PATH], {})).exit_code: + if time.monotonic() > deadline: + raise SandboxError(f"the relay did not start within {RELAY_START_SECONDS}s") + await asyncio.sleep(0.2) + return {"url": RELAY_URL} + + +async def reroute( + runtime: Runtime, relay: dict[str, Any], upstream: str, tokens: dict[str, str] +) -> None: + """Point a running relay (`start`) at `upstream`, swapping its box tokens for + `tokens`' values. Its requests in flight retry against the new upstream.""" + await _write_config(runtime, upstream, tokens) + result = await runtime.run(RELAY_RELOAD_COMMAND, {}) + if result.exit_code: + raise SandboxError(f"the relay did not reload: {result.stderr.strip()[-500:]}") + + +async def _write_config( + runtime: Runtime, upstream: str, tokens: dict[str, str] +) -> None: + await runtime.run(["mkdir", "-p", RELAY_DIR], {}) + # Written whole, then moved: the relay must never read a partial config. + staged = f"{RELAY_CONFIG_PATH}.next" + await runtime.write(staged, relay_config(upstream, tokens)) + result = await runtime.run( + ["sh", "-c", f"mv {shlex.quote(staged)} {shlex.quote(RELAY_CONFIG_PATH)}"], {} + ) + if result.exit_code: + raise SandboxError(f"writing the relay config failed: {result.stderr[-500:]}") diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py new file mode 100644 index 0000000000..03d3e11c67 --- /dev/null +++ b/verifiers/v1/interception/relay/server.py @@ -0,0 +1,622 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Loopback HTTP relay between a harness program in the box and the host's interception +server. + +The harness is launched against `http://127.0.0.1:` instead of the host's URL. The +relay forwards every request to the configured upstream and swaps the box's bearer +tokens for the upstream's. The upstream can change while the harness runs: the host +replaces a lost tunnel with a new URL. On SIGHUP the relay re-reads its config, aborts +every in-flight upstream request and retries each one against the new upstream, as +long as its client has seen nothing but the status, headers and SSE comments: a model +call still waiting for its first event. + +While the upstream cannot be reached (a connect fails, a request breaks before its +response, the tunnel edge answers with its own error page), the relay retries the +request for up to `retry_seconds`, so the harness does not see a tunnel that is +reconnecting or being replaced. A retry is marked as one (`x-stainless-retry-count`), +so the interception server joins it to the request it already serves. + +Usage: relay_server.py --config --port [--ready ] [--pid ] + +The config file is JSON: `{"upstream": "https://host/base", "tokens": {"": +""}, "retry_seconds": 300}`. Once listening, the relay writes its port to `--ready` and its +process id to `--pid`. +""" + +import argparse +import contextlib +import http.client +import json +import os +import signal +import socket +import ssl +import struct +import sys +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +KEEPALIVE_SECONDS = 10.0 +"""Interval of the comments an SSE client gets while its request waits for a retry.""" +RETRY_SECONDS = 5 * 60 +"""How long a request is retried while the upstream cannot be reached, unless the config +sets `retry_seconds`: long enough for the host to notice a lost tunnel, open a new one +and reroute the relay.""" +EDGE_STATUSES = frozenset({404, 502, 503, 504}) +"""Statuses a tunnel edge answers with its own page when it cannot reach the upstream +("404 Tunnel Not Found", a bad gateway). The interception server answers with JSON or +SSE, so only a response of another type with one of these statuses is the edge's.""" +BACKOFF_SECONDS = 0.5 +BACKOFF_CAP_SECONDS = 10.0 +CONNECT_TIMEOUT_SECONDS = 10.0 +CHUNK_BYTES = 64 * 1024 +RETRY_COUNT_HEADER = "x-stainless-retry-count" +HOP_BY_HOP = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) +# The relay sends the whole body itself and lets http.client set Host for the upstream. +REQUEST_DROPPED = HOP_BY_HOP | {"host", "content-length", "expect"} +# Not terminated by a blank line, like the interception server's own keepalives: some +# SSE clients try to decode the empty event a blank line would dispatch. +KEEPALIVE = b": relay keepalive\n" +STATUS_CHANGED = "relay: upstream changed status on retry" +SSL_CONTEXT = ssl.create_default_context() + + +def log(message: str) -> None: + print(f"relay: {message}", file=sys.stderr, flush=True) + + +def error_event(message: str) -> bytes: + return b"event: error\ndata: " + json.dumps({"error": message}).encode() + b"\n\n" + + +class Config: + def __init__( + self, + upstream: str, + tokens: dict[str, str], + retry_seconds: float = RETRY_SECONDS, + ) -> None: + parts = urlsplit(upstream) + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError(f"upstream must be an http(s) URL: {upstream!r}") + if not isinstance(tokens, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in tokens.items() + ): + raise ValueError("tokens must map strings to strings") + self.scheme = parts.scheme + self.host = parts.hostname + self.port = parts.port + self.prefix = parts.path.rstrip("/") + self.tokens = tokens + self.retry_seconds = float(retry_seconds) + port = f":{parts.port}" if parts.port else "" + self.origin = f"{parts.scheme}://{parts.hostname}{port}{self.prefix}" + + @classmethod + def load(cls, path: str) -> "Config": + with open(path) as file: + data = json.load(file) + return cls( + data["upstream"], + data.get("tokens") or {}, + data.get("retry_seconds", RETRY_SECONDS), + ) + + def connection(self) -> http.client.HTTPConnection: + if self.scheme == "https": + return http.client.HTTPSConnection( + self.host, + self.port, + timeout=CONNECT_TIMEOUT_SECONDS, + context=SSL_CONTEXT, + ) + return http.client.HTTPConnection( + self.host, self.port, timeout=CONNECT_TIMEOUT_SECONDS + ) + + def swap(self, name: str, value: str) -> str: + if name == "x-api-key": + return self.tokens.get(value, value) + scheme, _, token = value.partition(" ") + token = token.strip() + if scheme.lower() == "bearer" and token in self.tokens: + return f"{scheme} {self.tokens[token]}" + return value + + +class Upstream: + """One in-flight upstream attempt, abortable from another thread.""" + + def __init__(self) -> None: + self.lock = threading.Lock() + self.sock: socket.socket | None = None + self.aborted = False + + def attach(self, sock: socket.socket) -> None: + with self.lock: + self.sock = sock + aborted = self.aborted + if aborted: + shutdown(sock) + + def abort(self) -> None: + with self.lock: + self.aborted = True + sock = self.sock + if sock is not None: + shutdown(sock) + + +def shutdown(sock: socket.socket) -> None: + # Unlike `close`, `shutdown` wakes a read blocked in another thread. The base class + # method because `SSLSocket.shutdown` also drops the TLS state that read still uses. + with contextlib.suppress(OSError): + socket.socket.shutdown(sock, socket.SHUT_RDWR) + + +class Relay: + def __init__(self, path: str) -> None: + self.path = path + self.config = Config.load(path) + self.changed = threading.Condition() + self.upstreams: set[Upstream] = set() + + def begin(self) -> tuple[Upstream, Config]: + """Register an attempt on the current config, atomically with reading it, so a + reload aborts every attempt that still uses the config it replaces.""" + with self.changed: + upstream = Upstream() + self.upstreams.add(upstream) + return upstream, self.config + + def end(self, upstream: Upstream) -> None: + with self.changed: + self.upstreams.discard(upstream) + + def wait(self, config: Config, timeout: float) -> None: + with self.changed: + self.changed.wait_for(lambda: self.config is not config, timeout) + + def reload(self) -> None: + try: + config = Config.load(self.path) + except (OSError, ValueError, KeyError) as error: + # Aborting would only retry against the old upstream. + log(f"reload failed, keeping {self.config.origin}: {error}") + return + with self.changed: + self.config = config + aborted, self.upstreams = self.upstreams, set() + self.changed.notify_all() + for upstream in aborted: + upstream.abort() + log(f"reloaded: upstream {config.origin}, aborted {len(aborted)}") + + +class UpstreamError(Exception): + pass + + +def is_edge_error(response: http.client.HTTPResponse) -> bool: + """Whether `response` is the tunnel edge's own error page, not the upstream's answer.""" + content_type = (response.getheader("Content-Type") or "").lower() + return response.status in EDGE_STATUSES and not content_type.startswith( + ("application/json", "text/event-stream") + ) + + +class ClientGone(Exception): + pass + + +class Scanner: + """Whether a response body forwarded anything but SSE comment lines and blank lines. + A client skips those, so a retry's body can still follow them.""" + + def __init__(self) -> None: + self.sse = False + self.data = False + self.line_start = True + + def feed(self, chunk: bytes) -> None: + if self.data or not chunk: + return + if not self.sse: + self.data = True + return + for byte in chunk: + if self.line_start: + if byte in b"\r\n": + continue + if byte != ord(":"): + self.data = True + return + self.line_start = False + elif byte in b"\r\n": + self.line_start = True + + def line_break(self) -> bytes: + """What ends a comment line that a broken upstream left unfinished.""" + return b"" if self.line_start else b"\n" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server: "Server" + + def __getattr__(self, name: str): + # Every method is relayed: `BaseHTTPRequestHandler` dispatches to `do_`. + if name.startswith("do_"): + return self.relay + raise AttributeError(name) + + def log_message(self, format: str, *args) -> None: + log(format % args) + + def relay(self) -> None: + try: + self.body = self.read_body() + except (OSError, ValueError): + self.send_error(400, "relay: unreadable request body") + return + target = self.path + if not target.startswith("/"): + parts = urlsplit(target) + target = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") + self.target = target + dropped = REQUEST_DROPPED | { + name.strip().lower() + for name in self.headers.get("Connection", "").split(",") + } + self.forwarded = [ + (name, value) + for name, value in self.headers.items() + if name.lower() not in dropped + ] + try: + self.retry_count = int(self.headers.get(RETRY_COUNT_HEADER, 0)) + except ValueError: + self.retry_count = 0 + self.retries = 0 + self.failing_since: float | None = None + self.delay = BACKOFF_SECONDS + self.sent_status: int | None = None + self.sent_length: int | None = None + self.chunked = False + self.scanner = Scanner() + self.write_lock = threading.Lock() + relay = self.server.relay + try: + while True: + try: + with self.keepalive(): + upstream, connection, response = self.open() + except UpstreamError as error: + log(f"{self.command} {self.label}: {error}") + self.fail(str(error)) + return + try: + done = self.stream(upstream, response) + finally: + relay.end(upstream) + connection.close() + if done: + retried = f" after {self.retries} retries" if self.retries else "" + log(f"{self.command} {self.label} {self.sent_status}{retried}") + return + if self.scanner.data: + log(f"{self.command} {self.label}: aborted after data, closing") + self.reset() + return + if upstream.aborted: + self.retries += 1 + log(f"{self.command} {self.label}: aborted, retrying") + else: + try: + self.unreachable(relay.config, "the response broke") + except UpstreamError as error: + self.fail(str(error)) + return + except ClientGone: + log(f"{self.command} {self.label}: client went away") + self.close_connection = True + + @property + def label(self) -> str: + return self.target.partition("?")[0] + + def read_body(self) -> bytes: + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size = int(self.rfile.readline(65537).split(b";")[0], 16) + if size == 0: + while self.rfile.readline(65537) not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline(65537) + length = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(length) if length else b"" + + def upstream_headers(self, config: Config) -> dict[str, str]: + headers: dict[str, str] = {} + for name, value in self.forwarded: + lower = name.lower() + if lower in ("authorization", "x-api-key"): + value = config.swap(lower, value) + elif lower == RETRY_COUNT_HEADER and self.retries: + continue + headers[name] = f"{headers[name]}, {value}" if name in headers else value + if self.retries: + headers[RETRY_COUNT_HEADER] = str(self.retry_count + self.retries) + return headers + + def open( + self, + ) -> tuple[Upstream, http.client.HTTPConnection, http.client.HTTPResponse]: + relay = self.server.relay + while True: + upstream, config = relay.begin() + connection = config.connection() + try: + connection.connect() + assert connection.sock is not None + upstream.attach(connection.sock) + # Only the connect is bounded: a model call may wait long for its first + # byte. + connection.sock.settimeout(None) + connection.request( + self.command, + config.prefix + self.target, + self.body or None, + self.upstream_headers(config), + ) + response = connection.getresponse() + except (OSError, http.client.HTTPException) as error: + relay.end(upstream) + connection.close() + if upstream.aborted: + self.retries += 1 + log(f"{self.command} {self.label}: aborted, retrying") + continue + self.unreachable(config, f"{error!r}") + continue + if is_edge_error(response): + with contextlib.suppress(OSError, http.client.HTTPException): + response.read() + relay.end(upstream) + connection.close() + self.unreachable(config, f"tunnel edge answered {response.status}") + continue + return upstream, connection, response + + def unreachable(self, config: Config, reason: str) -> None: + """Wait before the next attempt while the upstream cannot be reached: until the + config changes, at most `retry_seconds` after the first failure.""" + now = time.monotonic() + if self.failing_since is None: + self.failing_since = now + self.delay = BACKOFF_SECONDS + if now - self.failing_since >= config.retry_seconds: + raise UpstreamError( + f"upstream unreachable for {config.retry_seconds:g}s: {reason}" + ) + log(f"{config.origin} unreachable ({reason}), retry in {self.delay}s") + self.retries += 1 + self.server.relay.wait(config, self.delay) + self.delay = min(self.delay * 2, BACKOFF_CAP_SECONDS) + + @contextlib.contextmanager + def keepalive(self) -> Iterator[None]: + """Keep an SSE client's idle timer from firing while its request waits for a + retry.""" + if self.sent_status is None or not self.scanner.sse: + yield + return + stop = threading.Event() + + def tick() -> None: + while not stop.wait(KEEPALIVE_SECONDS): + try: + self.send(self.scanner.line_break() + KEEPALIVE) + except ClientGone: + return + + thread = threading.Thread(target=tick, daemon=True) + thread.start() + try: + yield + finally: + stop.set() + thread.join() + + def stream(self, upstream: Upstream, response: http.client.HTTPResponse) -> bool: + """Forward one upstream response; False when a reload aborted it.""" + if self.sent_status is None: + self.start(response) + elif response.status != self.sent_status: + log( + f"{self.command} {self.label}: retry answered {response.status}, " + f"sent {self.sent_status}" + ) + if self.scanner.sse: + self.send(self.scanner.line_break() + error_event(STATUS_CHANGED)) + self.finish_body() + else: + self.reset() + return True + elif self.sent_length is not None and response.length != self.sent_length: + self.reset() + return True + else: + self.send(self.scanner.line_break()) + while True: + try: + data = response.read1(CHUNK_BYTES) + except (OSError, http.client.HTTPException) as error: + if upstream.aborted: + return False + log(f"{self.command} {self.label}: upstream broke: {error!r}") + if not self.scanner.data and self.scanner.sse: + # Nothing but comments went out: the retry can still answer. + return False + self.reset() + return True + # Data read after the abort is dropped: the retry sends it again. + if upstream.aborted: + return False + if not data: + self.finish_body() + return True + self.send(data) + + def start(self, response: http.client.HTTPResponse) -> None: + status = response.status + content_type = response.getheader("Content-Type", "") + encoding = response.getheader("Content-Encoding", "identity") + self.scanner.sse = content_type.startswith("text/event-stream") and ( + encoding.lower() == "identity" + ) + dropped = HOP_BY_HOP | {"content-length"} + dropped |= { + name.strip().lower() + for name in (response.getheader("Connection") or "").split(",") + } + self.send_response_only(status, response.reason) + for name, value in response.getheaders(): + if name.lower() not in dropped: + self.send_header(name, value) + bodyless = self.command == "HEAD" or status in (204, 304) or status < 200 + if bodyless: + if length := response.getheader("Content-Length"): + self.send_header("Content-Length", length) + elif response.length is not None and not response.chunked: + self.sent_length = response.length + self.send_header("Content-Length", str(response.length)) + elif self.request_version != "HTTP/1.0": + self.chunked = True + self.send_header("Transfer-Encoding", "chunked") + else: + self.close_connection = True + self.send_header("Connection", "close") + try: + self.end_headers() + except OSError as error: + raise ClientGone from error + self.sent_status = status + + def send(self, data: bytes) -> None: + if not data: + return + with self.write_lock: + self.scanner.feed(data) + if self.chunked: + data = b"%x\r\n%s\r\n" % (len(data), data) + try: + self.wfile.write(data) + self.wfile.flush() + except OSError as error: + raise ClientGone from error + + def finish_body(self) -> None: + if self.chunked: + with self.write_lock: + try: + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + except OSError as error: + raise ClientGone from error + + def fail(self, message: str) -> None: + if self.sent_status is None: + body = json.dumps({"error": f"relay: {message}"}).encode() + self.send_response_only(502) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + try: + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + except OSError as error: + raise ClientGone from error + elif self.scanner.sse: + self.send(self.scanner.line_break() + error_event(f"relay: {message}")) + self.finish_body() + else: + self.reset() + + def reset(self) -> None: + """Break the client's connection so it cannot take a cut body for a whole one.""" + self.close_connection = True + with contextlib.suppress(OSError): + self.connection.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0) + ) + + +class Server(ThreadingHTTPServer): + daemon_threads = True + request_queue_size = 128 + + def __init__(self, address: tuple[str, int], relay: Relay) -> None: + self.relay = relay + super().__init__(address, Handler) + + +def write_file(path: str, text: str) -> None: + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + tmp = f"{path}.{os.getpid()}.tmp" + with open(tmp, "w") as file: + file.write(text) + os.replace(tmp, path) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.partition("\n")[0]) + parser.add_argument("--config", required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--ready") + parser.add_argument("--pid") + args = parser.parse_args() + relay = Relay(args.config) + # Blocked before any thread starts, so threads inherit the mask and only the + # `sigwait` below takes SIGHUP: a reload then never interrupts a handler. + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGHUP}) + server = Server(("127.0.0.1", args.port), relay) + port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + if args.pid: + write_file(args.pid, f"{os.getpid()}\n") + if args.ready: + write_file(args.ready, f"{port}\n") + log(f"listening on 127.0.0.1:{port}, upstream {relay.config.origin}") + while True: + signal.sigwait({signal.SIGHUP}) + relay.reload() + + +if __name__ == "__main__": + main() diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 2ab70a0d15..d996234c87 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -364,6 +364,7 @@ def __init__( make_tunnel(self.config.tunnel) if requires_tunnel else None ) self.healthy = True + self._replacing: asyncio.Task | None = None if self.tunnel is not None: self.tunnel.on_gone = self._handle_tunnel_gone self.host = "127.0.0.1" @@ -377,12 +378,70 @@ def load(self) -> int: return len(self.sessions) def _handle_tunnel_gone(self) -> None: - """Fail this server's rollouts and take no more: their URL will never work again.""" + """Take no more rollouts: the URL will never work again. Rollouts that can move + (`RolloutSession.reroute`) follow this server to a new tunnel; the others fail.""" self.healthy = False logger.error("interception unhealthy: url=%s", self.base_url) + movable = False for session in list(self.sessions.values()): - session.error = TunnelError("interception tunnel lost") - session.release() + if session.reroute is None: + session.error = TunnelError("interception tunnel lost") + session.release() + else: + movable = True + if movable and (self._replacing is None or self._replacing.done()): + self._replacing = asyncio.create_task(self._replace_tunnel()) + + async def _replace_tunnel(self) -> None: + """Open a new tunnel to this server and reroute its movable rollouts to it. A + rollout that does not move fails as on a lost tunnel.""" + assert self.tunnel is not None + try: + base_url = await self.stack.enter_async_context( + self.tunnel.expose(self.port) + ) + except TunnelError as error: + logger.error("interception tunnel replacement failed: %s", error) + base_url = None + else: + logger.warning( + "interception tunnel replaced: url=%s -> %s", self.base_url, base_url + ) + self.base_url = base_url + + async def move(session: RolloutSession) -> None: + try: + if base_url is None: + raise TunnelError("interception tunnel lost") + assert session.reroute is not None + await session.reroute(base_url) + except Exception as error: # noqa: BLE001 - one rollout's failure is its own + logger.warning( + "rollout %s did not move to the new tunnel: %s", + session.trace.id, + error, + ) + session.error = ( + error + if isinstance(error, RolloutError) + else TunnelError(f"interception tunnel lost: {error}") + ) + session.release() + + await asyncio.gather( + *( + move(session) + for session in list(self.sessions.values()) + if session.reroute is not None and not session.released + ) + ) + + async def stop(self) -> None: + if self._replacing is not None: + self._replacing.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._replacing + await super().stop() def _client(self, config: BaseClientConfig) -> Client: """The server-owned client for `config` — one per distinct endpoint config, shared diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 4b83a53f73..233c6c4ef3 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -7,6 +7,7 @@ from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from typing import Any from urllib.parse import urlsplit from verifiers.v1.clients import ModelContext @@ -29,7 +30,7 @@ verifier_runtime, ) from verifiers.v1.harness import Harness, HarnessSession -from verifiers.v1.interception import Interception, serve_interception +from verifiers.v1.interception import Interception, relay, serve_interception from verifiers.v1.mcp import SharedToolServer, serve_tools from verifiers.v1.runtimes import ( Runtime, @@ -187,6 +188,8 @@ def __init__( self._closed = False self._harness_cleaned = False self._endpoint: str | None = None + self._relay: dict[str, Any] | None = None + """The relay the harness reaches the interception server through (`agent.relay`).""" self._urls: dict[str, str] = {} self._harness_session: HarnessSession | None = None self._agent_runtime: Runtime | None = None @@ -372,7 +375,17 @@ async def open(self) -> bool: self._shared_tools, ) ) - self._endpoint = runtime.host_url(f"{base_url.rstrip('/')}/v1") + harness_base = runtime.host_url(base_url.rstrip("/")) + if self.trace.agent.config.relay and not runtime.is_local: + harness_base = await self._start_relay( + runtime, + base_url, + {model_secret: model_secret, state_secret: state_secret}, + movable=not toolsets + and not self._shared_tools + and self.deployment.spec is None, + ) + self._endpoint = f"{harness_base}/v1" self._secret = model_secret services = ( { @@ -409,7 +422,11 @@ async def open(self) -> bool: await self.deployment.prepare_execution( [ url - for url in [self._endpoint, *self._urls.values()] + for url in [ + self._endpoint, + *self._urls.values(), + *([base_url] if self._relay is not None else []), + ] if urlsplit(url).hostname not in internal ], state_route=f"{base_url.rstrip('/')}/state", @@ -502,11 +519,7 @@ async def open(self) -> bool: ) if not self._session.stopped: session_kwargs = ( - { - "tool_interception_url": runtime.host_url( - f"{base_url.rstrip('/')}/tool" - ) - } + {"tool_interception_url": f"{harness_base}/tool"} if self.harness.SUPPORTS_TOOL_INTERCEPTION and ( self._session.request_interceptors @@ -551,6 +564,33 @@ async def open(self) -> bool: self.trace.notify() return not self._session.stopped + async def _start_relay( + self, + runtime: Runtime, + base_url: str, + tokens: dict[str, str], + *, + movable: bool, + ) -> str: + """Start the relay the harness reaches the interception server through, and return + its base URL in the box. A `movable` rollout follows its server to a replacement + tunnel (`RolloutSession.reroute`).""" + upstream = base_url.rstrip("/") + self._relay = await relay.start(runtime, self.trace.id, upstream, tokens) + if movable: + relay_info = self._relay + + async def reroute(new_base_url: str) -> None: + new_upstream = new_base_url.rstrip("/") + # A restricted box must reach the new tunnel host before the relay uses it. + await self.deployment.prepare_execution( + [self._endpoint, new_upstream], state_route=f"{new_upstream}/state" + ) + await relay.reroute(runtime, relay_info, new_upstream, tokens) + + self._session.reroute = reroute + return self._relay["url"] + async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed or not self.ok: raise RuntimeError("this rollout is closed or stopped") diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index a0ec7f120f..01d7ccb7e1 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -179,6 +179,10 @@ class RolloutSession: multiplexes shares one keepalive connection pool instead of opening its own.""" proxy_mcp: MCPProxy | None = field(default=None, repr=False) """Register a host MCP backend on this session's interception server while serving.""" + reroute: Callable[[str], Awaitable[None]] | None = field(default=None, repr=False) + """Point everything this rollout reaches the interception server through at a new + base URL (the server replaced its tunnel). None when the rollout cannot move: its + tunnel loss then fails it.""" error: "RolloutError | None" = None """The latest unresolved model-call failure. The harness only sees it as an HTTP error, so when its program dies on it the rollout records this original error instead of a secondary From d81764f1975c20568845fa2b34efaf5971ff2576 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 5 Nov 2026 02:20:41 +0000 Subject: [PATCH 02/53] always relay a remote harness and fix review findings Drop `agent.relay`: a harness in a remote box always reaches the interception server through its relay, and a relay that does not start leaves it on the tunnel URL with a warning. - Give the relay only the model secret, which the harness already holds: a harness that reads the relay config gains no capability. - Run one relay per rollout (own dir, ephemeral port) and stop it when the rollout closes, so rollouts that share a box never share one. - Make a rollout movable before the server registers it; a tunnel lost while the relay starts waits for it. - Replace the tunnel again when it is lost during a replacement. - Retry a non-streaming body that breaks before any data, and a body that ends before its Content-Length. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 4 +- tests/integration/test_relay.py | 18 ++- tests/unit/interception/test_relay.py | 29 +++++ tests/unit/interception/test_tunnel.py | 30 +++++ verifiers/v1/configs/agent.py | 7 -- verifiers/v1/interception/relay/__init__.py | 126 +++++++++++--------- verifiers/v1/interception/relay/server.py | 11 +- verifiers/v1/interception/server.py | 18 ++- verifiers/v1/rollout.py | 107 ++++++++++++----- 9 files changed, 241 insertions(+), 109 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index c1e033eb6e..ab78e8a0ff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -27,12 +27,12 @@ Routing traffic through the interception server enables several things beyond fo ### Relay -A Prime tunnel can drop for a while (its client reconnects) or for good (the tunnel service drops its registration). A lost tunnel fails every rollout on its interception server. With `[env.agent] relay = true`, a harness in a remote box reaches the interception server through a relay in its own box, at `127.0.0.1`, instead of the tunnel URL: +A Prime tunnel can drop for a while (its client reconnects) or for good (the tunnel service drops its registration). A harness in a remote box therefore reaches the interception server through a relay in its own box, at `127.0.0.1`, not at the tunnel URL: - While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. - When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. -A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay needs Python 3.10 or later in the box. +A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a Python script that verifiers runs with uv; setup installs uv and a Python when the image has none. A relay that does not start leaves the harness on the tunnel URL, with a warning. ## Finishing a rollout diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py index 535eceda85..6750de5d05 100644 --- a/tests/integration/test_relay.py +++ b/tests/integration/test_relay.py @@ -1,8 +1,9 @@ """The interception relay on Prime: the server's tunnel is lost while a rollout's model call is in flight: its registration is deleted and its frpc connection drops, so it -cannot log in again. With `agent.relay`, the server opens a new tunnel, reroutes the box's -relay to it, and the rollout finishes as if nothing happened: the waiting model call -goes through once, on one branch. Without the relay, the rollout fails, as before. +cannot log in again. The server opens a new tunnel, reroutes the box's relay to it, and +the rollout finishes as if nothing happened: the waiting model call goes through once, +on one branch. A rollout whose relay did not start reaches the server at the tunnel +URL and fails, as before the relay. The model is scripted and holds its second reply until the test has dropped the tunnel and, with the relay, until the server has replaced it.""" @@ -18,8 +19,10 @@ import verifiers.v1 as vf from verifiers.v1.dialects.chat import ChatDialect +from verifiers.v1.errors import SandboxError from verifiers.v1.harnesses.bash import BashHarnessConfig from verifiers.v1.interception import InterceptionServer +from verifiers.v1.interception import relay as relay_module from verifiers.v1.utils.prime import load_prime_config pytestmark = pytest.mark.skipif( @@ -149,6 +152,8 @@ def _cmdline(pid: int) -> str: async def _run(relay: bool) -> tuple[vf.Trace, HeldModel, str, str]: + """`relay=False` makes the relay fail to start, so the harness reaches the server + directly.""" async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: agent = vf.make_agent( vf.AgentConfig( @@ -156,7 +161,6 @@ async def _run(relay: bool) -> tuple[vf.Trace, HeldModel, str, str]: runtime=vf.PrimeConfig(image="python:3.11-slim", labels=["vf-ci"]), model="scripted", client=vf.EvalClientConfig(base_url=model.url), - relay=relay, ), interception=server, ) @@ -192,7 +196,11 @@ async def test_rollout_survives_a_replaced_tunnel(): assert len(trace.branches) == 1 -async def test_rollout_without_relay_fails_on_a_lost_tunnel(): +async def test_rollout_without_relay_fails_on_a_lost_tunnel(monkeypatch): + async def fail(*args, **kwargs): + raise SandboxError("no relay") + + monkeypatch.setattr(relay_module, "start", fail) trace, _, _, _ = await _run(relay=False) assert not trace.ok diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 3dd0b47a6f..5e407bb048 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -439,3 +439,32 @@ def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): assert response.read().endswith(b"data: from-b\n\n") retried = b.requests.get(timeout=TIMEOUT)["headers"]["x-stainless-retry-count"] assert int(retried) >= 1 + + +def test_body_cut_short_is_retried(upstreams, make_relay): + """An upstream that ends before its `Content-Length` (a tunnel that drops mid-body) + is retried while the harness has no body data yet.""" + a, _ = upstreams + body = json.dumps({"answer": "x" * 64}).encode() + attempts = [] + + def respond(handler) -> None: + attempts.append(True) + handler.send_response(200) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + if len(attempts) == 1: + handler.wfile.flush() + handler.close_connection = True + return + handler.wfile.write(body) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + response = connection.getresponse() + assert response.status == 200 + assert response.read() == body + assert len(attempts) == 2 diff --git a/tests/unit/interception/test_tunnel.py b/tests/unit/interception/test_tunnel.py index 60b60619b9..aceb093fd7 100644 --- a/tests/unit/interception/test_tunnel.py +++ b/tests/unit/interception/test_tunnel.py @@ -162,6 +162,36 @@ async def test_gone_tunnel_moves_movable_rollouts_to_a_new_tunnel(): await server.stop() +async def test_tunnel_lost_during_a_replacement_is_replaced_again(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.tunnel.urls = iter( + ["https://first.example", "https://second.example", "https://third.example"] + ) + server.base_url = await server.stack.enter_async_context(server.tunnel.expose(0)) + moved: list = [] + + async def reroute(url: str) -> None: + moved.append(url) + if len(moved) == 1: + # The new tunnel is lost while the rollouts are rerouted to it. + server._handle_tunnel_gone() + + session = SimpleNamespace( + error=None, released=False, reroute=reroute, trace=SimpleNamespace(id="m") + ) + session.release = lambda: setattr(session, "released", True) + server.sessions["m"] = session + + server._handle_tunnel_gone() + await server._replacing + + assert moved == ["https://second.example", "https://third.example"] + assert server.base_url == "https://third.example" + assert session.error is None + await server.stop() + + async def test_failed_replacement_fails_the_movable_rollouts(): server = InterceptionServer() server.tunnel = ReplacingTunnel() diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 7691863fe4..19d484563d 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -79,13 +79,6 @@ class AgentConfig(BaseConfig): network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" - relay: bool = False - """Reach the interception server through a relay in a remote box (127.0.0.1) instead - of its tunnel URL. The relay retries a model call while the tunnel reconnects, and - when a tunnel is lost for good, the server opens a new one and reroutes the relay, - so the rollout goes on. A rollout with tool servers or deployment services still - fails on a lost tunnel: they reach the old URL. Needs Python 3.10+ in the box.""" - @field_serializer("scoring") def _scoring_overrides( self, scoring: ScoringConfig, info: SerializationInfo diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 8e87e84dfb..1454bb75ec 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -1,19 +1,15 @@ """Host side of the in-box relay (`server.py`). -A harness in a remote box can reach the interception server through a loopback relay in -the box, at `RELAY_URL`, with the box tokens of the relay config as its secrets. The relay -forwards to the current upstream and swaps each box token for its upstream token. When -the upstream URL changes (the server replaced a lost tunnel), the host points it at the -new one: - -1. Write `relay_config(upstream, tokens)` to `RELAY_CONFIG_PATH`, creating `RELAY_DIR`. -2. Start `[*await runtime.prepare_uv_script(RELAY_SOURCE, activate=False), - *relay_args()]` in the background, and wait for `RELAY_READY_PATH` to exist. It - holds the listening port; `RELAY_PID_PATH` holds the process id. -3. To reroute, rewrite `RELAY_CONFIG_PATH` and run `RELAY_RELOAD_COMMAND`. The relay - re-reads the config, aborts every in-flight upstream request and retries each one - that has sent its client only the status, headers and SSE comments. A request that - already streamed data downstream has its client connection closed instead. +A harness in a remote box reaches the interception server through a loopback relay in +its box, with the box tokens of the relay config as its secrets. The relay forwards to +the current upstream and swaps each box token for its upstream token. When the upstream +URL changes (the server replaced a lost tunnel), the host points it at the new one. + +Each rollout runs its own relay, under `RELAY_ROOT/` and on a port of its own, so +rollouts that share a box never share a relay: `start` launches it, `reroute` rewrites +its config and makes it reload (SIGHUP), and `stop` ends it. On a reload the relay aborts +every in-flight upstream request and retries each one that has sent its client only the +status, headers and SSE comments. """ from __future__ import annotations @@ -21,6 +17,7 @@ import asyncio import shlex import time +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit @@ -33,21 +30,10 @@ from verifiers.v1.runtimes.runtime import Runtime RELAY_SOURCE = (Path(__file__).resolve().parent / "server.py").read_text() -RELAY_DIR = "/tmp/vf-relay" -RELAY_CONFIG_PATH = f"{RELAY_DIR}/config.json" -"""The relay's config: `RelayConfig` as JSON. It holds upstream secrets.""" -RELAY_READY_PATH = f"{RELAY_DIR}/port" -"""Written with the listening port (decimal, newline-terminated) once the relay accepts -connections.""" -RELAY_PID_PATH = f"{RELAY_DIR}/pid" -"""Written with the relay's process id before `RELAY_READY_PATH`.""" -RELAY_PORT = 47821 -"""A fixed port, so the harness URL is known before the relay starts. Port 0 picks a -free one instead, read back from `RELAY_READY_PATH`.""" -RELAY_URL = f"http://127.0.0.1:{RELAY_PORT}" -RELAY_RELOAD_COMMAND = ["sh", "-c", f'kill -HUP "$(cat {RELAY_PID_PATH})"'] -"""Make the relay re-read `RELAY_CONFIG_PATH` (SIGHUP). An invalid config is logged and -ignored, keeping the old upstream.""" +RELAY_ROOT = "/tmp/vf-relay" +"""Where each rollout's relay keeps its files (`Relay.dir`).""" +RELAY_START_SECONDS = 60 +"""How long a started relay gets to accept connections.""" class RelayConfig(BaseModel): @@ -74,7 +60,8 @@ def _http_url(cls, upstream: str) -> str: def relay_config(upstream: str, tokens: dict[str, str], **options: Any) -> bytes: - """The contents of `RELAY_CONFIG_PATH`; `options` sets the other `RelayConfig` fields.""" + """The contents of a relay's config file; `options` sets the other `RelayConfig` + fields.""" return ( RelayConfig(upstream=upstream, tokens=tokens, **options) .model_dump_json() @@ -82,58 +69,81 @@ def relay_config(upstream: str, tokens: dict[str, str], **options: Any) -> bytes ) -def relay_args( - port: int = RELAY_PORT, - config: str = RELAY_CONFIG_PATH, - ready: str = RELAY_READY_PATH, - pid: str = RELAY_PID_PATH, -) -> list[str]: - """The relay's arguments, after the argv of its prepared script.""" - return ["--config", config, "--port", str(port), "--ready", ready, "--pid", pid] +@dataclass(frozen=True) +class Relay: + """A rollout's relay in its box.""" + dir: str + url: str + """The base URL the harness reaches the interception server at.""" -RELAY_START_SECONDS = 60 -"""How long a started relay gets to accept connections.""" + @property + def config_path(self) -> str: + """`RelayConfig` as JSON. It holds the upstream secret.""" + return f"{self.dir}/config.json" + + @property + def pid_path(self) -> str: + return f"{self.dir}/pid" async def start( runtime: Runtime, name: str, upstream: str, tokens: dict[str, str] -) -> dict[str, Any]: - """Start the relay in `runtime`'s box, forwarding to `upstream` with `tokens`, and - return what a later process needs to reroute it (`reroute`). `name` labels its log.""" - await _write_config(runtime, upstream, tokens) +) -> Relay: + """Start a relay in `runtime`'s box under `RELAY_ROOT/name`, forwarding to + `upstream` with `tokens`, on a free port.""" + directory = f"{RELAY_ROOT}/{name}" + ready = f"{directory}/port" + await runtime.run(["sh", "-c", f"rm -rf {shlex.quote(directory)}"], {}) + relay = Relay(dir=directory, url="") + await _write_config(runtime, relay, upstream, tokens) program = await runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + args = ["--config", relay.config_path, "--port", "0", "--ready", ready] await runtime.run_background( - [*program, *relay_args()], {}, log=f"{RELAY_DIR}/{name}.log" + [*program, *args, "--pid", relay.pid_path], {}, log=f"{directory}/relay.log" ) deadline = time.monotonic() + RELAY_START_SECONDS - while (await runtime.run(["test", "-s", RELAY_READY_PATH], {})).exit_code: + while True: + port = (await runtime.run(["cat", ready], {})).stdout.strip() + if port.isdigit(): + return Relay(dir=directory, url=f"http://127.0.0.1:{port}") if time.monotonic() > deadline: raise SandboxError(f"the relay did not start within {RELAY_START_SECONDS}s") await asyncio.sleep(0.2) - return {"url": RELAY_URL} async def reroute( - runtime: Runtime, relay: dict[str, Any], upstream: str, tokens: dict[str, str] + runtime: Runtime, relay: Relay, upstream: str, tokens: dict[str, str] ) -> None: - """Point a running relay (`start`) at `upstream`, swapping its box tokens for - `tokens`' values. Its requests in flight retry against the new upstream.""" - await _write_config(runtime, upstream, tokens) - result = await runtime.run(RELAY_RELOAD_COMMAND, {}) + """Point a running relay at `upstream`, swapping its box tokens for `tokens`' + values. Its requests in flight retry against the new upstream.""" + await _write_config(runtime, relay, upstream, tokens) + pid = shlex.quote(relay.pid_path) + result = await runtime.run(["sh", "-c", f'kill -HUP "$(cat {pid})"'], {}) if result.exit_code: raise SandboxError(f"the relay did not reload: {result.stderr.strip()[-500:]}") +async def stop(runtime: Runtime, relay: Relay) -> None: + """End a relay and remove its files.""" + pid = shlex.quote(relay.pid_path) + await runtime.run( + [ + "sh", + "-c", + f'kill "$(cat {pid})" 2>/dev/null; rm -rf {shlex.quote(relay.dir)}', + ], + {}, + ) + + async def _write_config( - runtime: Runtime, upstream: str, tokens: dict[str, str] + runtime: Runtime, relay: Relay, upstream: str, tokens: dict[str, str] ) -> None: - await runtime.run(["mkdir", "-p", RELAY_DIR], {}) + await runtime.run(["mkdir", "-p", relay.dir], {}) # Written whole, then moved: the relay must never read a partial config. - staged = f"{RELAY_CONFIG_PATH}.next" + staged = f"{relay.config_path}.next" await runtime.write(staged, relay_config(upstream, tokens)) - result = await runtime.run( - ["sh", "-c", f"mv {shlex.quote(staged)} {shlex.quote(RELAY_CONFIG_PATH)}"], {} - ) + result = await runtime.run(["mv", staged, relay.config_path], {}) if result.exit_code: raise SandboxError(f"writing the relay config failed: {result.stderr[-500:]}") diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index 03d3e11c67..43fa5147cb 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -478,8 +478,8 @@ def stream(self, upstream: Upstream, response: http.client.HTTPResponse) -> bool if upstream.aborted: return False log(f"{self.command} {self.label}: upstream broke: {error!r}") - if not self.scanner.data and self.scanner.sse: - # Nothing but comments went out: the retry can still answer. + if not self.scanner.data: + # No body data went out: the retry can still answer. return False self.reset() return True @@ -487,6 +487,13 @@ def stream(self, upstream: Upstream, response: http.client.HTTPResponse) -> bool if upstream.aborted: return False if not data: + if response.length: + # The upstream ended before its `Content-Length`. + log(f"{self.command} {self.label}: upstream body cut short") + if not self.scanner.data: + return False + self.reset() + return True self.finish_body() return True self.send(data) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d996234c87..eeca79898d 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -365,6 +365,8 @@ def __init__( ) self.healthy = True self._replacing: asyncio.Task | None = None + self._lost_again = False + """A tunnel was lost while a replacement was under way: replace it once more.""" if self.tunnel is not None: self.tunnel.on_gone = self._handle_tunnel_gone self.host = "127.0.0.1" @@ -389,8 +391,20 @@ def _handle_tunnel_gone(self) -> None: session.release() else: movable = True - if movable and (self._replacing is None or self._replacing.done()): - self._replacing = asyncio.create_task(self._replace_tunnel()) + if not movable: + return + if self._replacing is None or self._replacing.done(): + self._replacing = asyncio.create_task(self._replace_tunnels()) + else: + self._lost_again = True + + async def _replace_tunnels(self) -> None: + """Replace the tunnel until no loss arrived while a replacement was under way.""" + while True: + self._lost_again = False + await self._replace_tunnel() + if not self._lost_again: + return async def _replace_tunnel(self) -> None: """Open a new tunnel to this server and reroute its movable rollouts to it. A diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 233c6c4ef3..976b18e4e6 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -7,7 +7,6 @@ from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass -from typing import Any from urllib.parse import urlsplit from verifiers.v1.clients import ModelContext @@ -20,6 +19,7 @@ SandboxError, TaskError, ToolsetError, + TunnelError, boundary, ) from verifiers.v1.grading import ( @@ -188,8 +188,14 @@ def __init__( self._closed = False self._harness_cleaned = False self._endpoint: str | None = None - self._relay: dict[str, Any] | None = None - """The relay the harness reaches the interception server through (`agent.relay`).""" + self._relay: relay.Relay | None = None + """The relay a harness in a remote box reaches the interception server through.""" + self._relay_upstream: str | None = None + """The interception server URL the relay forwards to.""" + self._relay_started = asyncio.Event() + """Set once the relay started, or failed to start.""" + self._relay_runtime: Runtime | None = None + self._relay_tokens: dict[str, str] = {} self._urls: dict[str, str] = {} self._harness_session: HarnessSession | None = None self._agent_runtime: Runtime | None = None @@ -362,6 +368,19 @@ async def open(self) -> bool: # The harness reaches the model at `{base_url}/v1`; tool servers reach this # rollout's `/state` + `/task` at `base_url` — it's universally reachable # (the interception is exposed whenever any consumer is remote). + # A harness in a remote box reaches the interception server through a + # relay in its box, which follows the server to a replacement tunnel. A + # rollout whose tool servers or services reach the server too cannot move. + relayed = not runtime.is_local + if ( + relayed + and not toolsets + and not self._shared_tools + and self.deployment.spec is None + ): + # Set before the server registers the session: a tunnel lost while + # the relay starts waits for it. + self._session.reroute = self._reroute ( base_url, model_secret, @@ -376,14 +395,10 @@ async def open(self) -> bool: ) ) harness_base = runtime.host_url(base_url.rstrip("/")) - if self.trace.agent.config.relay and not runtime.is_local: - harness_base = await self._start_relay( - runtime, - base_url, - {model_secret: model_secret, state_secret: state_secret}, - movable=not toolsets - and not self._shared_tools - and self.deployment.spec is None, + if relayed: + harness_base = ( + await self._start_relay(runtime, base_url, model_secret) + or harness_base ) self._endpoint = f"{harness_base}/v1" self._secret = model_secret @@ -425,7 +440,7 @@ async def open(self) -> bool: for url in [ self._endpoint, *self._urls.values(), - *([base_url] if self._relay is not None else []), + *([self._relay_upstream] if self._relay_upstream else []), ] if urlsplit(url).hostname not in internal ], @@ -565,31 +580,57 @@ async def open(self) -> bool: return not self._session.stopped async def _start_relay( - self, - runtime: Runtime, - base_url: str, - tokens: dict[str, str], - *, - movable: bool, - ) -> str: + self, runtime: Runtime, base_url: str, model_secret: str + ) -> str | None: """Start the relay the harness reaches the interception server through, and return - its base URL in the box. A `movable` rollout follows its server to a replacement - tunnel (`RolloutSession.reroute`).""" + its base URL in the box. A relay that does not start leaves the harness on the + server's URL, unable to follow it to a replacement tunnel.""" upstream = base_url.rstrip("/") - self._relay = await relay.start(runtime, self.trace.id, upstream, tokens) - if movable: - relay_info = self._relay + # The harness holds the model secret anyway; the relay gets nothing more, so a + # harness that reads its config gains no capability. + tokens = {model_secret: model_secret} + try: + self._relay = await relay.start(runtime, self.trace.id, upstream, tokens) + except Exception as error: # noqa: BLE001 - the rollout runs on without it + logger.warning( + "rollout %s: the relay did not start, the harness reaches the " + "interception server directly: %s", + self.trace.id, + error, + ) + self._session.reroute = None + return None + finally: + self._relay_started.set() + self._relay_upstream = upstream + self._relay_runtime = runtime + self._relay_tokens = tokens + self._stack.push_async_callback(self._stop_relay) + return self._relay.url - async def reroute(new_base_url: str) -> None: - new_upstream = new_base_url.rstrip("/") - # A restricted box must reach the new tunnel host before the relay uses it. - await self.deployment.prepare_execution( - [self._endpoint, new_upstream], state_route=f"{new_upstream}/state" - ) - await relay.reroute(runtime, relay_info, new_upstream, tokens) + async def _reroute(self, base_url: str) -> None: + """Point the harness's relay at the interception server's new URL + (`RolloutSession.reroute`).""" + await self._relay_started.wait() + if self._relay is None or self._relay_runtime is None: + raise TunnelError("the harness reaches the interception server directly") + upstream = base_url.rstrip("/") + # A restricted box must reach the new tunnel host before the relay uses it. + await self.deployment.prepare_execution( + [self._endpoint, upstream], state_route=f"{upstream}/state" + ) + await relay.reroute( + self._relay_runtime, self._relay, upstream, self._relay_tokens + ) + self._relay_upstream = upstream - self._session.reroute = reroute - return self._relay["url"] + async def _stop_relay(self) -> None: + if self._relay is None or self._relay_runtime is None: + return + try: + await relay.stop(self._relay_runtime, self._relay) + except Exception: # noqa: BLE001 - the box may be gone already + logger.debug("rollout %s: stopping the relay failed", self.trace.id) async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed or not self.ok: From cbd8410c4b0a470ef0b127c8c98d60393a797d60 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 5 Nov 2026 02:42:09 +0000 Subject: [PATCH 03/53] start the relay alongside the harness setup The relay added about 4 s to setup (GSM8K, p50 6.4 s -> 10.6 s): eight sandbox calls one after another. Launch it while the harness sets up, with a placeholder upstream, and configure it once the interception slot is acquired: one write and one exec on the critical path. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_relay.py | 2 +- verifiers/v1/interception/relay/__init__.py | 46 +++++++++------------ verifiers/v1/rollout.py | 44 ++++++++++++++------ 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py index 6750de5d05..7641ff07d2 100644 --- a/tests/integration/test_relay.py +++ b/tests/integration/test_relay.py @@ -200,7 +200,7 @@ async def test_rollout_without_relay_fails_on_a_lost_tunnel(monkeypatch): async def fail(*args, **kwargs): raise SandboxError("no relay") - monkeypatch.setattr(relay_module, "start", fail) + monkeypatch.setattr(relay_module, "launch", fail) trace, _, _, _ = await _run(relay=False) assert not trace.ok diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 1454bb75ec..903b70c618 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -6,8 +6,8 @@ URL changes (the server replaced a lost tunnel), the host points it at the new one. Each rollout runs its own relay, under `RELAY_ROOT/` and on a port of its own, so -rollouts that share a box never share a relay: `start` launches it, `reroute` rewrites -its config and makes it reload (SIGHUP), and `stop` ends it. On a reload the relay aborts +rollouts that share a box never share a relay: `launch` starts it before its upstream is +known, `reroute` writes its config and makes it reload (SIGHUP), and `stop` ends it. On a reload the relay aborts every in-flight upstream request and retries each one that has sent its client only the status, headers and SSE comments. """ @@ -17,6 +17,7 @@ import asyncio import shlex import time +import uuid from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -87,16 +88,16 @@ def pid_path(self) -> str: return f"{self.dir}/pid" -async def start( - runtime: Runtime, name: str, upstream: str, tokens: dict[str, str] -) -> Relay: - """Start a relay in `runtime`'s box under `RELAY_ROOT/name`, forwarding to - `upstream` with `tokens`, on a free port.""" +async def launch(runtime: Runtime, name: str) -> Relay: + """Start a relay in `runtime`'s box under `RELAY_ROOT/name`, on a free port, before + its upstream is known: it forwards nowhere until `reroute` configures it.""" directory = f"{RELAY_ROOT}/{name}" - ready = f"{directory}/port" - await runtime.run(["sh", "-c", f"rm -rf {shlex.quote(directory)}"], {}) relay = Relay(dir=directory, url="") - await _write_config(runtime, relay, upstream, tokens) + # Each launch reads its port from a file of its own: a relay an earlier attempt + # left in the box can never answer for this one. + ready = f"{directory}/port-{uuid.uuid4().hex}" + # The gateway creates the directory. Nothing listens on port 9 (discard). + await runtime.write(relay.config_path, relay_config("http://127.0.0.1:9", {})) program = await runtime.prepare_uv_script(RELAY_SOURCE, activate=False) args = ["--config", relay.config_path, "--port", "0", "--ready", ready] await runtime.run_background( @@ -109,7 +110,7 @@ async def start( return Relay(dir=directory, url=f"http://127.0.0.1:{port}") if time.monotonic() > deadline: raise SandboxError(f"the relay did not start within {RELAY_START_SECONDS}s") - await asyncio.sleep(0.2) + await asyncio.sleep(0.1) async def reroute( @@ -117,9 +118,14 @@ async def reroute( ) -> None: """Point a running relay at `upstream`, swapping its box tokens for `tokens`' values. Its requests in flight retry against the new upstream.""" - await _write_config(runtime, relay, upstream, tokens) - pid = shlex.quote(relay.pid_path) - result = await runtime.run(["sh", "-c", f'kill -HUP "$(cat {pid})"'], {}) + # Written whole, then moved: the relay must never read a partial config. + staged = f"{relay.config_path}.next" + await runtime.write(staged, relay_config(upstream, tokens)) + config, pid = shlex.quote(relay.config_path), shlex.quote(relay.pid_path) + result = await runtime.run( + ["sh", "-c", f'mv {shlex.quote(staged)} {config} && kill -HUP "$(cat {pid})"'], + {}, + ) if result.exit_code: raise SandboxError(f"the relay did not reload: {result.stderr.strip()[-500:]}") @@ -135,15 +141,3 @@ async def stop(runtime: Runtime, relay: Relay) -> None: ], {}, ) - - -async def _write_config( - runtime: Runtime, relay: Relay, upstream: str, tokens: dict[str, str] -) -> None: - await runtime.run(["mkdir", "-p", relay.dir], {}) - # Written whole, then moved: the relay must never read a partial config. - staged = f"{relay.config_path}.next" - await runtime.write(staged, relay_config(upstream, tokens)) - result = await runtime.run(["mv", staged, relay.config_path], {}) - if result.exit_code: - raise SandboxError(f"writing the relay config failed: {result.stderr[-500:]}") diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 976b18e4e6..2aead6512d 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -196,6 +196,7 @@ def __init__( """Set once the relay started, or failed to start.""" self._relay_runtime: Runtime | None = None self._relay_tokens: dict[str, str] = {} + self._relay_launch: asyncio.Task[relay.Relay] | None = None self._urls: dict[str, str] = {} self._harness_session: HarnessSession | None = None self._agent_runtime: Runtime | None = None @@ -363,15 +364,23 @@ async def open(self) -> bool: self._agent_runtime = await resolve_agent_runtime( runtime, self.task.data.agent_user ) + # A harness in a remote box reaches the interception server through + # a relay in its box, which follows the server to a replacement + # tunnel. It starts alongside the harness setup. + relayed = not runtime.is_local + if relayed: + self._relay_runtime = runtime + self._relay_launch = asyncio.create_task( + relay.launch(runtime, self.trace.id) + ) + self._stack.push_async_callback(self._stop_relay) await self.harness.setup(self._agent_runtime) # `base_url` is the interception server's reachable URL for this rollout. # The harness reaches the model at `{base_url}/v1`; tool servers reach this # rollout's `/state` + `/task` at `base_url` — it's universally reachable # (the interception is exposed whenever any consumer is remote). - # A harness in a remote box reaches the interception server through a - # relay in its box, which follows the server to a replacement tunnel. A - # rollout whose tool servers or services reach the server too cannot move. - relayed = not runtime.is_local + # A rollout whose tool servers or services reach the server too cannot + # follow it to a replacement tunnel. if ( relayed and not toolsets @@ -582,15 +591,17 @@ async def open(self) -> bool: async def _start_relay( self, runtime: Runtime, base_url: str, model_secret: str ) -> str | None: - """Start the relay the harness reaches the interception server through, and return - its base URL in the box. A relay that does not start leaves the harness on the - server's URL, unable to follow it to a replacement tunnel.""" + """Point the launched relay at the interception server, and return the base URL + the harness reaches it at in the box. A relay that does not start leaves the + harness on the server's URL, unable to follow it to a replacement tunnel.""" + assert self._relay_launch is not None upstream = base_url.rstrip("/") # The harness holds the model secret anyway; the relay gets nothing more, so a # harness that reads its config gains no capability. tokens = {model_secret: model_secret} try: - self._relay = await relay.start(runtime, self.trace.id, upstream, tokens) + launched = await self._relay_launch + await relay.reroute(runtime, launched, upstream, tokens) except Exception as error: # noqa: BLE001 - the rollout runs on without it logger.warning( "rollout %s: the relay did not start, the harness reaches the " @@ -602,11 +613,10 @@ async def _start_relay( return None finally: self._relay_started.set() + self._relay = launched self._relay_upstream = upstream - self._relay_runtime = runtime self._relay_tokens = tokens - self._stack.push_async_callback(self._stop_relay) - return self._relay.url + return launched.url async def _reroute(self, base_url: str) -> None: """Point the harness's relay at the interception server's new URL @@ -625,10 +635,18 @@ async def _reroute(self, base_url: str) -> None: self._relay_upstream = upstream async def _stop_relay(self) -> None: - if self._relay is None or self._relay_runtime is None: + # A reroute still waiting for the relay finds none. + self._relay_started.set() + launch, runtime = self._relay_launch, self._relay_runtime + if launch is None or runtime is None: + return + if not launch.done(): + launch.cancel() + await asyncio.wait({launch}) + if launch.cancelled() or launch.exception() is not None: return try: - await relay.stop(self._relay_runtime, self._relay) + await relay.stop(runtime, launch.result()) except Exception: # noqa: BLE001 - the box may be gone already logger.debug("rollout %s: stopping the relay failed", self.trace.id) From c05642fc2328fed42011d8e2a6c0577318d1de04 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 00:48:39 +0000 Subject: [PATCH 04/53] simplify the relay wiring and trust certifi's roots A `Relay` object owns the relay's lifecycle (launch alongside the harness setup, connect, reroute, stop), so the rollout keeps one field. The retry window is a constant; tests patch the script source. The relay verified the tunnel's certificate against the system CA store, which a task image may lack: terminal-bench-2's financial-document-processor image has no /etc/ssl/certs, so both its relayed rollouts failed on their first model call while the harness's own HTTP client (certifi) worked on main. Trust certifi's roots as well. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_relay.py | 2 +- tests/unit/interception/test_relay.py | 12 +- verifiers/v1/interception/relay/__init__.py | 212 ++++++++++---------- verifiers/v1/interception/relay/server.py | 38 ++-- verifiers/v1/rollout.py | 115 +++-------- 5 files changed, 156 insertions(+), 223 deletions(-) diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py index 7641ff07d2..00757a357d 100644 --- a/tests/integration/test_relay.py +++ b/tests/integration/test_relay.py @@ -200,7 +200,7 @@ async def test_rollout_without_relay_fails_on_a_lost_tunnel(monkeypatch): async def fail(*args, **kwargs): raise SandboxError("no relay") - monkeypatch.setattr(relay_module, "launch", fail) + monkeypatch.setattr(relay_module.Relay, "connect", fail) trace, _, _, _ = await _run(relay=False) assert not trace.ok diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 5e407bb048..9aa1a38564 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -127,9 +127,7 @@ def __init__( upstream: str, tokens: dict[str, str], source: str, - retry_seconds: float = 300.0, ) -> None: - self.retry_seconds = retry_seconds script = tmp_path / "relay_server.py" script.write_text(source) self.config = tmp_path / "config.json" @@ -144,9 +142,7 @@ def __init__( self.port = int(wait_for(lambda: ready.exists() and ready.read_text())) def write(self, upstream: str, tokens: dict[str, str]) -> None: - self.config.write_bytes( - relay_config(upstream, tokens, retry_seconds=self.retry_seconds) - ) + self.config.write_bytes(relay_config(upstream, tokens)) def reload(self, upstream: str, tokens: dict[str, str]) -> None: self.write(upstream, tokens) @@ -197,9 +193,8 @@ def make( upstream: str, tokens: dict[str, str] | None = None, source: str = RELAY_SOURCE, - retry_seconds: float = 300.0, ) -> Relay: - relays.append(Relay(tmp_path, upstream, tokens or {}, source, retry_seconds)) + relays.append(Relay(tmp_path, upstream, tokens or {}, source)) return relays[-1] yield make @@ -358,7 +353,8 @@ def call() -> None: def test_unreachable_upstream_answers_502_after_the_retry_window(make_relay): - relay = make_relay("http://127.0.0.1:1", retry_seconds=1) + source = RELAY_SOURCE.replace("RETRY_SECONDS = 5 * 60", "RETRY_SECONDS = 1") + relay = make_relay("http://127.0.0.1:1", source=source) started = time.monotonic() response = sse_post(relay) assert response.status == 502 diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 903b70c618..eec39426be 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -1,143 +1,139 @@ """Host side of the in-box relay (`server.py`). -A harness in a remote box reaches the interception server through a loopback relay in -its box, with the box tokens of the relay config as its secrets. The relay forwards to -the current upstream and swaps each box token for its upstream token. When the upstream -URL changes (the server replaced a lost tunnel), the host points it at the new one. +A harness in a remote box reaches the interception server through a relay that runs in +its box, at `127.0.0.1`. The relay forwards every request to its current upstream and +swaps the harness's bearer tokens for the upstream's, so the host can move the harness +to a new URL or new secrets without the harness noticing: `Relay.reroute`. Each rollout runs its own relay, under `RELAY_ROOT/` and on a port of its own, so -rollouts that share a box never share a relay: `launch` starts it before its upstream is -known, `reroute` writes its config and makes it reload (SIGHUP), and `stop` ends it. On a reload the relay aborts -every in-flight upstream request and retries each one that has sent its client only the -status, headers and SSE comments. +rollouts that share a box never share one. """ from __future__ import annotations import asyncio +import contextlib +import json import shlex import time import uuid -from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any -from urllib.parse import urlsplit +from typing import TYPE_CHECKING -from pydantic import BaseModel, ConfigDict, field_validator - -from verifiers.v1.errors import SandboxError +from verifiers.v1.errors import SandboxError, TunnelError if TYPE_CHECKING: from verifiers.v1.runtimes.runtime import Runtime RELAY_SOURCE = (Path(__file__).resolve().parent / "server.py").read_text() RELAY_ROOT = "/tmp/vf-relay" -"""Where each rollout's relay keeps its files (`Relay.dir`).""" RELAY_START_SECONDS = 60 -"""How long a started relay gets to accept connections.""" - - -class RelayConfig(BaseModel): - """The relay's config file.""" - - model_config = ConfigDict(extra="forbid") +"""How long a launched relay gets to accept connections.""" - upstream: str - """Base URL that request paths are appended to; it may carry a path prefix.""" - tokens: dict[str, str] = {} - """Box token to upstream token, swapped in `Authorization: Bearer` and `x-api-key`.""" - retry_seconds: float = 300.0 - """How long the relay retries a request while the upstream cannot be reached.""" - @field_validator("upstream") - @classmethod - def _http_url(cls, upstream: str) -> str: - parts = urlsplit(upstream) - if parts.scheme not in ("http", "https") or not parts.hostname: - raise ValueError(f"upstream must be an http(s) URL: {upstream!r}") - if parts.query or parts.fragment: - raise ValueError(f"upstream must not have a query: {upstream!r}") - return upstream +def relay_config(upstream: str, tokens: dict[str, str]) -> bytes: + """A relay's config file: the upstream base URL, and the harness's tokens mapped to + the upstream's.""" + return json.dumps({"upstream": upstream, "tokens": tokens}).encode() -def relay_config(upstream: str, tokens: dict[str, str], **options: Any) -> bytes: - """The contents of a relay's config file; `options` sets the other `RelayConfig` - fields.""" - return ( - RelayConfig(upstream=upstream, tokens=tokens, **options) - .model_dump_json() - .encode() - ) - - -@dataclass(frozen=True) class Relay: - """A rollout's relay in its box.""" - - dir: str - url: str - """The base URL the harness reaches the interception server at.""" + """A rollout's relay in its box. It launches when created, before its upstream is + known, so its start overlaps the harness setup; `connect` points it at the + interception server, `reroute` moves it, `stop` ends it.""" + + def __init__(self, runtime: Runtime, name: str) -> None: + self.runtime = runtime + self.dir = f"{RELAY_ROOT}/{name}" + self.url = "" + """The base URL the harness reaches the interception server at, once launched.""" + self.upstream = "" + self.tokens: dict[str, str] = {} + self._connected = asyncio.Event() + self._launch = asyncio.create_task(self._start()) @property def config_path(self) -> str: - """`RelayConfig` as JSON. It holds the upstream secret.""" return f"{self.dir}/config.json" @property def pid_path(self) -> str: return f"{self.dir}/pid" - -async def launch(runtime: Runtime, name: str) -> Relay: - """Start a relay in `runtime`'s box under `RELAY_ROOT/name`, on a free port, before - its upstream is known: it forwards nowhere until `reroute` configures it.""" - directory = f"{RELAY_ROOT}/{name}" - relay = Relay(dir=directory, url="") - # Each launch reads its port from a file of its own: a relay an earlier attempt - # left in the box can never answer for this one. - ready = f"{directory}/port-{uuid.uuid4().hex}" - # The gateway creates the directory. Nothing listens on port 9 (discard). - await runtime.write(relay.config_path, relay_config("http://127.0.0.1:9", {})) - program = await runtime.prepare_uv_script(RELAY_SOURCE, activate=False) - args = ["--config", relay.config_path, "--port", "0", "--ready", ready] - await runtime.run_background( - [*program, *args, "--pid", relay.pid_path], {}, log=f"{directory}/relay.log" - ) - deadline = time.monotonic() + RELAY_START_SECONDS - while True: - port = (await runtime.run(["cat", ready], {})).stdout.strip() - if port.isdigit(): - return Relay(dir=directory, url=f"http://127.0.0.1:{port}") - if time.monotonic() > deadline: - raise SandboxError(f"the relay did not start within {RELAY_START_SECONDS}s") - await asyncio.sleep(0.1) - - -async def reroute( - runtime: Runtime, relay: Relay, upstream: str, tokens: dict[str, str] -) -> None: - """Point a running relay at `upstream`, swapping its box tokens for `tokens`' - values. Its requests in flight retry against the new upstream.""" - # Written whole, then moved: the relay must never read a partial config. - staged = f"{relay.config_path}.next" - await runtime.write(staged, relay_config(upstream, tokens)) - config, pid = shlex.quote(relay.config_path), shlex.quote(relay.pid_path) - result = await runtime.run( - ["sh", "-c", f'mv {shlex.quote(staged)} {config} && kill -HUP "$(cat {pid})"'], - {}, - ) - if result.exit_code: - raise SandboxError(f"the relay did not reload: {result.stderr.strip()[-500:]}") - - -async def stop(runtime: Runtime, relay: Relay) -> None: - """End a relay and remove its files.""" - pid = shlex.quote(relay.pid_path) - await runtime.run( - [ - "sh", - "-c", - f'kill "$(cat {pid})" 2>/dev/null; rm -rf {shlex.quote(relay.dir)}', - ], - {}, - ) + async def _start(self) -> None: + # Each launch reads its port from a file of its own: a relay an earlier attempt + # left in the box can never answer for this one. + ready = f"{self.dir}/port-{uuid.uuid4().hex}" + # The gateway creates the directory. Nothing listens on port 9 (discard). + await self.runtime.write( + self.config_path, relay_config("http://127.0.0.1:9", {}) + ) + program = await self.runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + args = ["--config", self.config_path, "--port", "0", "--ready", ready] + await self.runtime.run_background( + [*program, *args, "--pid", self.pid_path], {}, log=f"{self.dir}/relay.log" + ) + deadline = time.monotonic() + RELAY_START_SECONDS + while True: + port = (await self.runtime.run(["cat", ready], {})).stdout.strip() + if port.isdigit(): + self.url = f"http://127.0.0.1:{port}" + return + if time.monotonic() > deadline: + raise SandboxError( + f"the relay did not start within {RELAY_START_SECONDS}s" + ) + await asyncio.sleep(0.1) + + async def connect(self, upstream: str, tokens: dict[str, str]) -> str: + """Point the relay at the interception server once it runs, and return the base + URL the harness reaches the server at.""" + try: + await self._launch + await self._configure(upstream, tokens) + finally: + self._connected.set() + return self.url + + async def wait_connected(self) -> None: + """Wait until `connect` finished; raise if the relay does not forward.""" + await self._connected.wait() + if not self.upstream: + raise TunnelError("the harness reaches the interception server directly") + + async def reroute( + self, upstream: str, tokens: dict[str, str] | None = None + ) -> None: + """Move the relay to `upstream`, with `tokens` if the secrets changed. Its + requests in flight retry against the new upstream.""" + await self.wait_connected() + await self._configure(upstream, self.tokens if tokens is None else tokens) + + async def _configure(self, upstream: str, tokens: dict[str, str]) -> None: + # Written whole, then moved: the relay must never read a partial config. + staged = f"{self.config_path}.next" + await self.runtime.write(staged, relay_config(upstream, tokens)) + config, pid = shlex.quote(self.config_path), shlex.quote(self.pid_path) + reload = f'mv {shlex.quote(staged)} {config} && kill -HUP "$(cat {pid})"' + result = await self.runtime.run(["sh", "-c", reload], {}) + if result.exit_code: + raise SandboxError( + f"the relay did not reload: {result.stderr.strip()[-500:]}" + ) + self.upstream, self.tokens = upstream, tokens + + async def stop(self) -> None: + """End the relay and remove its files; best effort, the box may be gone.""" + self._connected.set() + if not self._launch.done(): + self._launch.cancel() + await asyncio.wait({self._launch}) + if self._launch.cancelled() or self._launch.exception() is not None: + return + pid, directory = shlex.quote(self.pid_path), shlex.quote(self.dir) + # Best effort: the box may be gone already. + with contextlib.suppress(Exception): + await self.runtime.run( + ["sh", "-c", f'kill "$(cat {pid})" 2>/dev/null; rm -rf {directory}'], + {}, + ) diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index 43fa5147cb..5c71fc60fe 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.10" -# dependencies = [] +# dependencies = ["certifi"] # /// """Loopback HTTP relay between a harness program in the box and the host's interception server. @@ -15,14 +15,14 @@ While the upstream cannot be reached (a connect fails, a request breaks before its response, the tunnel edge answers with its own error page), the relay retries the -request for up to `retry_seconds`, so the harness does not see a tunnel that is +request for up to `RETRY_SECONDS`, so the harness does not see a tunnel that is reconnecting or being replaced. A retry is marked as one (`x-stainless-retry-count`), so the interception server joins it to the request it already serves. Usage: relay_server.py --config --port [--ready ] [--pid ] The config file is JSON: `{"upstream": "https://host/base", "tokens": {"": -""}, "retry_seconds": 300}`. Once listening, the relay writes its port to `--ready` and its +""}}`. Once listening, the relay writes its port to `--ready` and its process id to `--pid`. """ @@ -42,12 +42,13 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import urlsplit +import certifi + KEEPALIVE_SECONDS = 10.0 """Interval of the comments an SSE client gets while its request waits for a retry.""" RETRY_SECONDS = 5 * 60 -"""How long a request is retried while the upstream cannot be reached, unless the config -sets `retry_seconds`: long enough for the host to notice a lost tunnel, open a new one -and reroute the relay.""" +"""How long a request is retried while the upstream cannot be reached: long enough for +the host to notice a lost tunnel, open a new one and reroute the relay.""" EDGE_STATUSES = frozenset({404, 502, 503, 504}) """Statuses a tunnel edge answers with its own page when it cannot reach the upstream ("404 Tunnel Not Found", a bad gateway). The interception server answers with JSON or @@ -77,7 +78,10 @@ # SSE clients try to decode the empty event a blank line would dispatch. KEEPALIVE = b": relay keepalive\n" STATUS_CHANGED = "relay: upstream changed status on retry" -SSL_CONTEXT = ssl.create_default_context() +# certifi's roots, as the harness's own HTTP clients use: a task image may have no system +# CA store (no `ca-certificates`), and the stdlib then trusts no certificate at all. +SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) +SSL_CONTEXT.load_default_certs() def log(message: str) -> None: @@ -89,12 +93,7 @@ def error_event(message: str) -> bytes: class Config: - def __init__( - self, - upstream: str, - tokens: dict[str, str], - retry_seconds: float = RETRY_SECONDS, - ) -> None: + def __init__(self, upstream: str, tokens: dict[str, str]) -> None: parts = urlsplit(upstream) if parts.scheme not in ("http", "https") or not parts.hostname: raise ValueError(f"upstream must be an http(s) URL: {upstream!r}") @@ -108,7 +107,6 @@ def __init__( self.port = parts.port self.prefix = parts.path.rstrip("/") self.tokens = tokens - self.retry_seconds = float(retry_seconds) port = f":{parts.port}" if parts.port else "" self.origin = f"{parts.scheme}://{parts.hostname}{port}{self.prefix}" @@ -116,11 +114,7 @@ def __init__( def load(cls, path: str) -> "Config": with open(path) as file: data = json.load(file) - return cls( - data["upstream"], - data.get("tokens") or {}, - data.get("retry_seconds", RETRY_SECONDS), - ) + return cls(data["upstream"], data.get("tokens") or {}) def connection(self) -> http.client.HTTPConnection: if self.scheme == "https": @@ -413,14 +407,14 @@ def open( def unreachable(self, config: Config, reason: str) -> None: """Wait before the next attempt while the upstream cannot be reached: until the - config changes, at most `retry_seconds` after the first failure.""" + config changes, at most `RETRY_SECONDS` after the first failure.""" now = time.monotonic() if self.failing_since is None: self.failing_since = now self.delay = BACKOFF_SECONDS - if now - self.failing_since >= config.retry_seconds: + if now - self.failing_since >= RETRY_SECONDS: raise UpstreamError( - f"upstream unreachable for {config.retry_seconds:g}s: {reason}" + f"upstream unreachable for {RETRY_SECONDS:g}s: {reason}" ) log(f"{config.origin} unreachable ({reason}), retry in {self.delay}s") self.retries += 1 diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 2aead6512d..55bd05e18d 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -19,7 +19,6 @@ SandboxError, TaskError, ToolsetError, - TunnelError, boundary, ) from verifiers.v1.grading import ( @@ -30,7 +29,8 @@ verifier_runtime, ) from verifiers.v1.harness import Harness, HarnessSession -from verifiers.v1.interception import Interception, relay, serve_interception +from verifiers.v1.interception import Interception, serve_interception +from verifiers.v1.interception.relay import Relay from verifiers.v1.mcp import SharedToolServer, serve_tools from verifiers.v1.runtimes import ( Runtime, @@ -188,15 +188,8 @@ def __init__( self._closed = False self._harness_cleaned = False self._endpoint: str | None = None - self._relay: relay.Relay | None = None + self._relay: Relay | None = None """The relay a harness in a remote box reaches the interception server through.""" - self._relay_upstream: str | None = None - """The interception server URL the relay forwards to.""" - self._relay_started = asyncio.Event() - """Set once the relay started, or failed to start.""" - self._relay_runtime: Runtime | None = None - self._relay_tokens: dict[str, str] = {} - self._relay_launch: asyncio.Task[relay.Relay] | None = None self._urls: dict[str, str] = {} self._harness_session: HarnessSession | None = None self._agent_runtime: Runtime | None = None @@ -365,30 +358,24 @@ async def open(self) -> bool: runtime, self.task.data.agent_user ) # A harness in a remote box reaches the interception server through - # a relay in its box, which follows the server to a replacement - # tunnel. It starts alongside the harness setup. - relayed = not runtime.is_local - if relayed: - self._relay_runtime = runtime - self._relay_launch = asyncio.create_task( - relay.launch(runtime, self.trace.id) - ) - self._stack.push_async_callback(self._stop_relay) + # a relay in its box, which starts alongside the harness setup. + if not runtime.is_local: + self._relay = Relay(runtime, self.trace.id) + self._stack.push_async_callback(self._relay.stop) await self.harness.setup(self._agent_runtime) # `base_url` is the interception server's reachable URL for this rollout. # The harness reaches the model at `{base_url}/v1`; tool servers reach this # rollout's `/state` + `/task` at `base_url` — it's universally reachable # (the interception is exposed whenever any consumer is remote). - # A rollout whose tool servers or services reach the server too cannot - # follow it to a replacement tunnel. + # The relay follows the server to a replacement tunnel, unless tool + # servers or services reach the server too. Set before the server + # registers the session: a tunnel lost while the relay starts waits. if ( - relayed + self._relay is not None and not toolsets and not self._shared_tools and self.deployment.spec is None ): - # Set before the server registers the session: a tunnel lost while - # the relay starts waits for it. self._session.reroute = self._reroute ( base_url, @@ -404,11 +391,21 @@ async def open(self) -> bool: ) ) harness_base = runtime.host_url(base_url.rstrip("/")) - if relayed: - harness_base = ( - await self._start_relay(runtime, base_url, model_secret) - or harness_base - ) + if self._relay is not None: + try: + # The harness holds the model secret anyway; the relay gets + # nothing more, so reading its config grants no capability. + harness_base = await self._relay.connect( + base_url.rstrip("/"), {model_secret: model_secret} + ) + except Exception as error: # noqa: BLE001 - runs on without it + logger.warning( + "rollout %s: the relay did not start, the harness reaches " + "the interception server directly: %s", + self.trace.id, + error, + ) + self._session.reroute = None self._endpoint = f"{harness_base}/v1" self._secret = model_secret services = ( @@ -449,7 +446,7 @@ async def open(self) -> bool: for url in [ self._endpoint, *self._urls.values(), - *([self._relay_upstream] if self._relay_upstream else []), + *([self._relay.upstream] if self._relay else []), ] if urlsplit(url).hostname not in internal ], @@ -588,67 +585,17 @@ async def open(self) -> bool: self.trace.notify() return not self._session.stopped - async def _start_relay( - self, runtime: Runtime, base_url: str, model_secret: str - ) -> str | None: - """Point the launched relay at the interception server, and return the base URL - the harness reaches it at in the box. A relay that does not start leaves the - harness on the server's URL, unable to follow it to a replacement tunnel.""" - assert self._relay_launch is not None - upstream = base_url.rstrip("/") - # The harness holds the model secret anyway; the relay gets nothing more, so a - # harness that reads its config gains no capability. - tokens = {model_secret: model_secret} - try: - launched = await self._relay_launch - await relay.reroute(runtime, launched, upstream, tokens) - except Exception as error: # noqa: BLE001 - the rollout runs on without it - logger.warning( - "rollout %s: the relay did not start, the harness reaches the " - "interception server directly: %s", - self.trace.id, - error, - ) - self._session.reroute = None - return None - finally: - self._relay_started.set() - self._relay = launched - self._relay_upstream = upstream - self._relay_tokens = tokens - return launched.url - async def _reroute(self, base_url: str) -> None: - """Point the harness's relay at the interception server's new URL + """Move the harness's relay to the interception server's new URL (`RolloutSession.reroute`).""" - await self._relay_started.wait() - if self._relay is None or self._relay_runtime is None: - raise TunnelError("the harness reaches the interception server directly") + assert self._relay is not None + await self._relay.wait_connected() upstream = base_url.rstrip("/") # A restricted box must reach the new tunnel host before the relay uses it. await self.deployment.prepare_execution( [self._endpoint, upstream], state_route=f"{upstream}/state" ) - await relay.reroute( - self._relay_runtime, self._relay, upstream, self._relay_tokens - ) - self._relay_upstream = upstream - - async def _stop_relay(self) -> None: - # A reroute still waiting for the relay finds none. - self._relay_started.set() - launch, runtime = self._relay_launch, self._relay_runtime - if launch is None or runtime is None: - return - if not launch.done(): - launch.cancel() - await asyncio.wait({launch}) - if launch.cancelled() or launch.exception() is not None: - return - try: - await relay.stop(runtime, launch.result()) - except Exception: # noqa: BLE001 - the box may be gone already - logger.debug("rollout %s: stopping the relay failed", self.trace.id) + await self._relay.reroute(upstream) async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed or not self.ok: From a2a293694072c911791ad232a5ecfa79fb9784cc Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 00:53:19 +0000 Subject: [PATCH 05/53] fail the rollout when its relay does not start The relay is part of the machinery like the interception server: a relay that does not start fails the rollout with a SandboxError, which the rollout retry rules cover, instead of a silent fallback to the tunnel URL. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- tests/integration/test_relay.py | 44 ++++----------------- verifiers/v1/interception/relay/__init__.py | 4 +- verifiers/v1/rollout.py | 14 ++----- 4 files changed, 14 insertions(+), 50 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index ab78e8a0ff..6984096ece 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,7 +32,7 @@ A Prime tunnel can drop for a while (its client reconnects) or for good (the tun - While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. - When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. -A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a Python script that verifiers runs with uv; setup installs uv and a Python when the image has none. A relay that does not start leaves the harness on the tunnel URL, with a warning. +A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a Python script that verifiers runs with uv; setup installs uv and a Python when the image has none. A relay that does not start fails the rollout with a `SandboxError`. ## Finishing a rollout diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py index 00757a357d..f283e991c7 100644 --- a/tests/integration/test_relay.py +++ b/tests/integration/test_relay.py @@ -2,11 +2,10 @@ call is in flight: its registration is deleted and its frpc connection drops, so it cannot log in again. The server opens a new tunnel, reroutes the box's relay to it, and the rollout finishes as if nothing happened: the waiting model call goes through once, -on one branch. A rollout whose relay did not start reaches the server at the tunnel -URL and fails, as before the relay. +on one branch. The model is scripted and holds its second reply until the test has dropped the tunnel -and, with the relay, until the server has replaced it.""" +and the server has replaced it.""" import asyncio import json @@ -19,10 +18,8 @@ import verifiers.v1 as vf from verifiers.v1.dialects.chat import ChatDialect -from verifiers.v1.errors import SandboxError from verifiers.v1.harnesses.bash import BashHarnessConfig from verifiers.v1.interception import InterceptionServer -from verifiers.v1.interception import relay as relay_module from verifiers.v1.utils.prime import load_prime_config pytestmark = pytest.mark.skipif( @@ -151,9 +148,7 @@ def _cmdline(pid: int) -> str: return "" -async def _run(relay: bool) -> tuple[vf.Trace, HeldModel, str, str]: - """`relay=False` makes the relay fail to start, so the harness reaches the server - directly.""" +async def test_rollout_survives_a_replaced_tunnel(): async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: agent = vf.make_agent( vf.AgentConfig( @@ -168,24 +163,14 @@ async def _run(relay: bool) -> tuple[vf.Trace, HeldModel, str, str]: await asyncio.wait_for(model.held.wait(), 600) first_url = server.base_url await drop_tunnel(first_url) - if relay: - # Hold the reply until the server serves the rollout through a new tunnel. - async with asyncio.timeout(DETECTION_SECONDS): - while server.base_url == first_url: - await asyncio.sleep(1) - else: - async with asyncio.timeout(DETECTION_SECONDS): - while server.healthy: - await asyncio.sleep(1) + # Hold the reply until the server serves the rollout through a new tunnel. + async with asyncio.timeout(DETECTION_SECONDS): + while server.base_url == first_url: + await asyncio.sleep(1) model.release.set() trace = await asyncio.wait_for(run, 600) - return trace, model, first_url, server.base_url - - -async def test_rollout_survives_a_replaced_tunnel(): - trace, model, first_url, last_url = await _run(relay=True) - assert last_url != first_url + assert server.base_url != first_url assert trace.ok, [error.message for error in trace.errors] assert trace.errors == [] tool_results = [str(m.content) for m in trace.messages if m.role == "tool"] @@ -194,16 +179,3 @@ async def test_rollout_survives_a_replaced_tunnel(): # server was already serving. assert model.requests[1] == 1 assert len(trace.branches) == 1 - - -async def test_rollout_without_relay_fails_on_a_lost_tunnel(monkeypatch): - async def fail(*args, **kwargs): - raise SandboxError("no relay") - - monkeypatch.setattr(relay_module.Relay, "connect", fail) - trace, _, _, _ = await _run(relay=False) - - assert not trace.ok - # The harness gets the tunnel edge's 404 and exits, usually before the server - # declares the tunnel gone and fails the rollout itself. - assert {error.type for error in trace.errors} & {"HarnessError", "TunnelError"} diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index eec39426be..676c189b25 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -96,10 +96,10 @@ async def connect(self, upstream: str, tokens: dict[str, str]) -> str: return self.url async def wait_connected(self) -> None: - """Wait until `connect` finished; raise if the relay does not forward.""" + """Wait until `connect` finished; raise if the relay did not start.""" await self._connected.wait() if not self.upstream: - raise TunnelError("the harness reaches the interception server directly") + raise TunnelError("the relay did not start") async def reroute( self, upstream: str, tokens: dict[str, str] | None = None diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 55bd05e18d..5c7a1f9642 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -392,20 +392,12 @@ async def open(self) -> bool: ) harness_base = runtime.host_url(base_url.rstrip("/")) if self._relay is not None: - try: - # The harness holds the model secret anyway; the relay gets - # nothing more, so reading its config grants no capability. + # The harness holds the model secret anyway; the relay gets + # nothing more, so reading its config grants no capability. + async with boundary(SandboxError, "starting the relay"): harness_base = await self._relay.connect( base_url.rstrip("/"), {model_secret: model_secret} ) - except Exception as error: # noqa: BLE001 - runs on without it - logger.warning( - "rollout %s: the relay did not start, the harness reaches " - "the interception server directly: %s", - self.trace.id, - error, - ) - self._session.reroute = None self._endpoint = f"{harness_base}/v1" self._secret = model_secret services = ( From 7b3f706b342c54ee2738b65b26cb72eed5cf6daf Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 00:56:52 +0000 Subject: [PATCH 06/53] name the relay in the trace when it fails Add `RelayError` (an `InterceptionError`, next to `TunnelError`). A relay that does not start raises it. A relay that gives up on a request appends the reason to a failures file in its directory; when the harness then dies, the rollout checks that file and whether the relay still runs, and records `RelayError` with the relay's reason instead of the harness's `HarnessError`. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_relay.py | 40 ++++++++++++++++-- tests/unit/interception/test_relay.py | 45 ++++++++++++++++++++- verifiers/v1/__init__.py | 3 ++ verifiers/v1/errors.py | 5 +++ verifiers/v1/interception/relay/__init__.py | 25 +++++++++++- verifiers/v1/interception/relay/server.py | 22 ++++++++-- verifiers/v1/rollout.py | 6 ++- 7 files changed, 137 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py index f283e991c7..0d61c8cc1e 100644 --- a/tests/integration/test_relay.py +++ b/tests/integration/test_relay.py @@ -2,10 +2,10 @@ call is in flight: its registration is deleted and its frpc connection drops, so it cannot log in again. The server opens a new tunnel, reroutes the box's relay to it, and the rollout finishes as if nothing happened: the waiting model call goes through once, -on one branch. +on one branch. A relay that dies is named in the trace (`RelayError`), not the harness +that died on it. -The model is scripted and holds its second reply until the test has dropped the tunnel -and the server has replaced it.""" +The model is scripted and holds its second reply while the test breaks something.""" import asyncio import json @@ -20,6 +20,7 @@ from verifiers.v1.dialects.chat import ChatDialect from verifiers.v1.harnesses.bash import BashHarnessConfig from verifiers.v1.interception import InterceptionServer +from verifiers.v1.interception.relay import Relay from verifiers.v1.utils.prime import load_prime_config pytestmark = pytest.mark.skipif( @@ -179,3 +180,36 @@ async def test_rollout_survives_a_replaced_tunnel(): # server was already serving. assert model.requests[1] == 1 assert len(trace.branches) == 1 + + +async def test_a_dead_relay_is_named_in_the_trace(monkeypatch): + """The relay exits while a model call waits: the harness dies on the dropped + connection, and the trace records `RelayError`, not `HarnessError`.""" + relays: list[Relay] = [] + connect = Relay.connect + + async def recording(self, upstream, tokens): + relays.append(self) + return await connect(self, upstream, tokens) + + monkeypatch.setattr(Relay, "connect", recording) + async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: + agent = vf.make_agent( + vf.AgentConfig( + harness=BashHarnessConfig(id="bash"), + runtime=vf.PrimeConfig(image="python:3.11-slim", labels=["vf-ci"]), + model="scripted", + client=vf.EvalClientConfig(base_url=model.url), + ), + interception=server, + ) + run = asyncio.create_task(agent.run(vf.Task(vf.TaskData(idx=0, prompt="Go.")))) + await asyncio.wait_for(model.held.wait(), 600) + (relay,) = relays + await relay.runtime.run(["sh", "-c", f'kill -9 "$(cat {relay.pid_path})"'], {}) + trace = await asyncio.wait_for(run, 600) + model.release.set() + + assert not trace.ok + assert [error.type for error in trace.errors] == ["RelayError"] + assert "the relay exited" in trace.errors[0].message diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 9aa1a38564..418e7d12ed 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -2,6 +2,7 @@ responses as they arrive, and on SIGHUP moves requests still waiting for their first event to the new upstream on the same client connection.""" +import asyncio import http.client import json import os @@ -15,6 +16,7 @@ from collections.abc import Callable, Iterator from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from types import SimpleNamespace import pytest @@ -133,10 +135,12 @@ def __init__( self.config = tmp_path / "config.json" self.write(upstream, tokens) ready = tmp_path / "port" + self.failures = tmp_path / "failures" self.stderr = (tmp_path / "relay.log").open("w+") self.process = subprocess.Popen( [sys.executable, str(script), "--config", str(self.config)] - + ["--port", "0", "--ready", str(ready)], + + ["--port", "0", "--ready", str(ready)] + + ["--failures", str(self.failures)], stderr=self.stderr, ) self.port = int(wait_for(lambda: ready.exists() and ready.read_text())) @@ -360,6 +364,8 @@ def test_unreachable_upstream_answers_502_after_the_retry_window(make_relay): assert response.status == 502 assert "unreachable for 1s" in json.loads(response.read())["error"] assert 1 <= time.monotonic() - started < 5 + # The host reads the reason back to name the relay, not the harness, in the trace. + assert "unreachable for 1s" in relay.failures.read_text() def edge_page(status: int): @@ -464,3 +470,40 @@ def respond(handler) -> None: assert response.status == 200 assert response.read() == body assert len(attempts) == 2 + + +class FakeRuntime: + """Answers the relay's failure check with a fixed stdout.""" + + def __init__(self, stdout: str) -> None: + self.stdout = stdout + + async def run(self, argv, env): + return SimpleNamespace(stdout=self.stdout, exit_code=0) + + +@pytest.mark.parametrize( + ("stdout", "expected"), + [ + ( + "upstream unreachable for 300s: refused\n", + "upstream unreachable for 300s: refused", + ), + ("the relay exited\n", "the relay exited"), + ("", None), + ], + ids=["gave-up", "exited", "healthy"], +) +async def test_relay_failure_names_the_relay(monkeypatch, stdout, expected): + from verifiers.v1.errors import RelayError + from verifiers.v1.interception.relay import Relay + + monkeypatch.setattr(Relay, "_start", lambda self: asyncio.sleep(0)) + relay = Relay(FakeRuntime(stdout), "trace") + relay.upstream = "https://tunnel.example" + failure = await relay.failure() + if expected is None: + assert failure is None + else: + assert isinstance(failure, RelayError) and str(failure) == expected + await relay.stop() diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index ecaf8b4d6e..378cf584b1 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -65,6 +65,7 @@ HarnessError, InterceptionError, ProviderError, + RelayError, RolloutError, SandboxError, TaskError, @@ -326,6 +327,7 @@ "TrainRunInfo": "verifiers.v1.trace", "TrainWorkInfo": "verifiers.v1.trace", "TunnelError": "verifiers.v1.errors", + "RelayError": "verifiers.v1.errors", "TurnTokens": "verifiers.v1.types", "Usage": "verifiers.v1.types", "UserMessage": "verifiers.v1.types", @@ -456,6 +458,7 @@ "TaskError", "InterceptionError", "TunnelError", + "RelayError", # clients "Client", "BaseClientConfig", diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index d581748e45..6ff8959ec9 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -78,6 +78,11 @@ class TunnelError(InterceptionError): or was lost.""" +class RelayError(InterceptionError): + """The relay in the box between the harness and the host interception server failed: + it did not start, gave up on a request, or exited.""" + + @contextlib.asynccontextmanager async def boundary(error_cls: type[RolloutError], what: str) -> AsyncIterator[None]: """Run a framework→code boundary, attributing any error escaping it to `error_cls`. An diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 676c189b25..29fb52008e 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from verifiers.v1.errors import SandboxError, TunnelError +from verifiers.v1.errors import RelayError, SandboxError, TunnelError if TYPE_CHECKING: from verifiers.v1.runtimes.runtime import Runtime @@ -60,6 +60,11 @@ def config_path(self) -> str: def pid_path(self) -> str: return f"{self.dir}/pid" + @property + def failures_path(self) -> str: + """Where the relay appends the reason for each request it gives up on.""" + return f"{self.dir}/failures" + async def _start(self) -> None: # Each launch reads its port from a file of its own: a relay an earlier attempt # left in the box can never answer for this one. @@ -70,6 +75,7 @@ async def _start(self) -> None: ) program = await self.runtime.prepare_uv_script(RELAY_SOURCE, activate=False) args = ["--config", self.config_path, "--port", "0", "--ready", ready] + args += ["--failures", self.failures_path] await self.runtime.run_background( [*program, *args, "--pid", self.pid_path], {}, log=f"{self.dir}/relay.log" ) @@ -122,6 +128,23 @@ async def _configure(self, upstream: str, tokens: dict[str, str]) -> None: ) self.upstream, self.tokens = upstream, tokens + async def failure(self) -> RelayError | None: + """The relay's failure, if it gave up on a request or exited; for a harness that + died, so the trace names the relay rather than the harness.""" + if not self.upstream: + return None + failures, pid = shlex.quote(self.failures_path), shlex.quote(self.pid_path) + check = ( + f"tail -n 1 {failures} 2>/dev/null; " + f'kill -0 "$(cat {pid})" 2>/dev/null || echo "the relay exited"' + ) + with contextlib.suppress(Exception): + result = await self.runtime.run(["sh", "-c", check], {}) + reasons = result.stdout.strip().splitlines() + if reasons: + return RelayError("; ".join(reasons)) + return None + async def stop(self) -> None: """End the relay and remove its files; best effort, the box may be gone.""" self._connected.set() diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index 5c71fc60fe..dfbe58bfdf 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -20,10 +20,12 @@ so the interception server joins it to the request it already serves. Usage: relay_server.py --config --port [--ready ] [--pid ] + [--failures ] The config file is JSON: `{"upstream": "https://host/base", "tokens": {"": ""}}`. Once listening, the relay writes its port to `--ready` and its -process id to `--pid`. +process id to `--pid`. Each request it gives up on appends its reason to `--failures`, +so the host can tell a failure of the relay from one of the harness. """ import argparse @@ -169,8 +171,10 @@ def shutdown(sock: socket.socket) -> None: class Relay: - def __init__(self, path: str) -> None: + def __init__(self, path: str, failures: str | None = None) -> None: self.path = path + self.failures = failures + self.failures_lock = threading.Lock() self.config = Config.load(path) self.changed = threading.Condition() self.upstreams: set[Upstream] = set() @@ -183,6 +187,16 @@ def begin(self) -> tuple[Upstream, Config]: self.upstreams.add(upstream) return upstream, self.config + def record_failure(self, reason: str) -> None: + if self.failures is None: + return + with ( + self.failures_lock, + contextlib.suppress(OSError), + open(self.failures, "a") as file, + ): + file.write(reason.replace("\n", " ") + "\n") + def end(self, upstream: Upstream) -> None: with self.changed: self.upstreams.discard(upstream) @@ -550,6 +564,7 @@ def finish_body(self) -> None: raise ClientGone from error def fail(self, message: str) -> None: + self.server.relay.record_failure(message) if self.sent_status is None: body = json.dumps({"error": f"relay: {message}"}).encode() self.send_response_only(502) @@ -601,8 +616,9 @@ def main() -> None: parser.add_argument("--port", type=int, required=True) parser.add_argument("--ready") parser.add_argument("--pid") + parser.add_argument("--failures") args = parser.parse_args() - relay = Relay(args.config) + relay = Relay(args.config, args.failures) # Blocked before any thread starts, so threads inherit the mask and only the # `sigwait` below takes SIGHUP: a reload then never interrupts a handler. signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGHUP}) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 5c7a1f9642..c5a4d9d49a 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -15,6 +15,7 @@ from verifiers.v1.configs.scoring import resolve_scoring from verifiers.v1.errors import ( HarnessError, + RelayError, RolloutError, SandboxError, TaskError, @@ -394,7 +395,7 @@ async def open(self) -> bool: if self._relay is not None: # The harness holds the model secret anyway; the relay gets # nothing more, so reading its config grants no capability. - async with boundary(SandboxError, "starting the relay"): + async with boundary(RelayError, "starting the relay"): harness_base = await self._relay.connect( base_url.rstrip("/"), {model_secret: model_secret} ) @@ -657,6 +658,9 @@ async def step( self.timeout("agent") return False real = self._session.error + if real is None and self._relay is not None: + # A harness that dies on its relay's failure records the relay's. + real = await self._relay.failure() if real is not None and isinstance(e, RolloutError): real.__cause__ = e self.fail(real) From 1c621001dea47a34b8888dd70b3ef8e46a42fee3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 00:58:20 +0000 Subject: [PATCH 07/53] set the relay field on the rollout test stub Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..70259729b0 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -45,6 +45,7 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._agent_time_remaining = time_remaining run._timeouts = RolloutTimeouts() run._harness_session = session + run._relay = None run.deployment = SimpleNamespace(run=lambda turn: turn) run._session = SimpleNamespace( request_interceptors=[], From d72d875d6404d7b5e064ebe920b69c7cb0baed98 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 01:34:12 +0000 Subject: [PATCH 08/53] bootstrap uv once per box at a time Scripts prepared in parallel each ran the uv bootstrap under a lock of their own script: in an image without uv, two `pip install --user uv` runs or downloads to one path (or apt's lock) raced, and one failed. The relay prepares its script during the harness setup, so CI hit it: a random network-policy cell failed with zero branches. The first preparation per user now holds a bootstrap lock; once uv is there, scripts prepare in parallel. The integration grid's one-branch assertions show the trace's errors. Co-Authored-By: Claude Opus 5.5 --- tests/integration/grid.py | 5 ++++- tests/integration/test_shared_tool.py | 4 +++- verifiers/v1/runtimes/base.py | 5 +++++ verifiers/v1/runtimes/runtime.py | 18 +++++++++++++++--- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 2d636ccd32..de5a3c605b 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -94,7 +94,10 @@ def assert_one_branch( f"no FORKS reason for {set(forks) - FORKS.keys()}" ) if harness.id not in forks: - assert trace.num_branches == 1, f"{trace.num_branches} branches" + assert trace.num_branches == 1, ( + f"{trace.num_branches} branches", + [f"{error.type}: {error.message[:500]}" for error in trace.errors], + ) def agent_seats(taskset: str, env_id: str = "") -> list[str]: diff --git a/tests/integration/test_shared_tool.py b/tests/integration/test_shared_tool.py index ddfd7f928c..e36dbaeaae 100644 --- a/tests/integration/test_shared_tool.py +++ b/tests/integration/test_shared_tool.py @@ -22,7 +22,9 @@ async def test_shared_tool(run, harness_runtime, tool_runtime): taskset_overrides={"tools": tool_runtime}, ) traces = [trace for episode in episodes for trace in episode.traces] - assert all(trace.num_branches == 1 for trace in traces) + assert all(trace.num_branches == 1 for trace in traces), [ + f"{error.type}: {error.message[:500]}" for t in traces for error in t.errors + ] assert len(traces) == 2 for trace in traces: assert trace.ok, [error.message for error in trace.errors] diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index b0c1d32756..2832876eff 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -147,6 +147,11 @@ class RuntimeCache: uv_interpreters: dict[str, str] = field(default_factory=dict) uv_locks: dict[str, asyncio.Lock] = field(default_factory=dict) + uv_ready: set[str] = field(default_factory=set) + """Users (`""` for the runtime's own) whose uv bootstrap succeeded.""" + uv_bootstrap_lock: asyncio.Lock = field(default_factory=asyncio.Lock) + """Held by a script preparation that may bootstrap uv: two bootstraps at once race + (concurrent `pip install --user`, downloads to one path, apt's lock).""" curl_ready: bool = False mcp_sources: set[str] = field(default_factory=set) mcp_lock: asyncio.Lock = field(default_factory=asyncio.Lock) diff --git a/verifiers/v1/runtimes/runtime.py b/verifiers/v1/runtimes/runtime.py index 628bc9b0cd..a7965fa0a6 100644 --- a/verifiers/v1/runtimes/runtime.py +++ b/verifiers/v1/runtimes/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import shlex import uuid @@ -179,9 +180,20 @@ async def prepare_uv_script( ) # Bootstrap dependencies before task networking is restricted; # keep task offline settings on the returned program's processes. - result = await self.run( - ["sh", "-c", command], {**(env or {}), "UV_OFFLINE": "false"} - ) + # Scripts prepare in parallel once uv is there; until then one at + # a time, so only one bootstraps it. + cache, user = self.backend.cache, self.user or "" + async with ( + contextlib.nullcontext() + if user in cache.uv_ready + else cache.uv_bootstrap_lock + ): + result = await self.run( + ["sh", "-c", command], + {**(env or {}), "UV_OFFLINE": "false"}, + ) + if result.exit_code == 0: + cache.uv_ready.add(user) if result.exit_code != 0: raise RuntimeError( "failed to prepare uv script: " From 19974a4a50b72c4a0aa0ca5db1fb5db8acd0f802 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 01:34:12 +0000 Subject: [PATCH 09/53] fix relay review findings - Mark a resend as a retry only after an attempt that may have reached the server; a connect failure or an edge page did not, so a body a later turn repeats is never answered from the replay cache. - Never frame SSE with a fixed length, and close an edge error page unread instead of draining it. - Count a new outage from its start once a response stayed up. - Record "ok" after a request goes through again, so a stale relay failure never names a later, unrelated harness death. - Stop the relay process also after a failed or cancelled launch. - Leave a closing rollout's egress alone on a reroute (shared scoring sets its own), and strip the trailing slash `host_url` may return. Co-Authored-By: Claude Opus 5.5 --- tests/unit/interception/test_relay.py | 26 ++++++++++--- verifiers/v1/interception/relay/__init__.py | 6 +-- verifiers/v1/interception/relay/server.py | 41 +++++++++++++++++---- verifiers/v1/rollout.py | 6 ++- 4 files changed, 63 insertions(+), 16 deletions(-) diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 418e7d12ed..5c6e3ae00a 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -368,6 +368,20 @@ def test_unreachable_upstream_answers_502_after_the_retry_window(make_relay): assert "unreachable for 1s" in relay.failures.read_text() +def test_a_request_through_after_a_failure_clears_it(upstreams, make_relay): + a, _ = upstreams + a.respond = lambda handler: reply(handler, 200, {"ok": True}) + source = RELAY_SOURCE.replace("RETRY_SECONDS = 5 * 60", "RETRY_SECONDS = 1") + relay = make_relay("http://127.0.0.1:1", source=source) + assert sse_post(relay).status == 502 + relay.reload(a.url, {}) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + assert connection.getresponse().status == 200 + # The last line is the relay's current state: it forwards again. + assert wait_for(lambda: relay.failures.read_text().splitlines()[-1] == "ok") + + def edge_page(status: int): """A tunnel edge's own error page.""" @@ -405,8 +419,9 @@ def respond(handler) -> None: a.requests.get(timeout=TIMEOUT)["headers"].get("x-stainless-retry-count") for _ in range(3) ] - # Each retry is marked, so the interception server joins it to the request it serves. - assert retries == [None, "1", "2"] + # The edge's pages mean the server never saw the request, so the resends are + # not marked as retries: a repeated body is never answered from its replay cache. + assert retries == [None, None, None] def test_interception_errors_reach_the_harness(upstreams, make_relay): @@ -439,8 +454,8 @@ def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): response = connection.getresponse() assert response.status == 200 assert response.read().endswith(b"data: from-b\n\n") - retried = b.requests.get(timeout=TIMEOUT)["headers"]["x-stainless-retry-count"] - assert int(retried) >= 1 + # The dead upstream never saw the request: the first one to arrive is no retry. + assert "x-stainless-retry-count" not in b.requests.get(timeout=TIMEOUT)["headers"] def test_body_cut_short_is_retried(upstreams, make_relay): @@ -490,9 +505,10 @@ async def run(self, argv, env): "upstream unreachable for 300s: refused", ), ("the relay exited\n", "the relay exited"), + ("ok\n", None), ("", None), ], - ids=["gave-up", "exited", "healthy"], + ids=["gave-up", "exited", "recovered", "healthy"], ) async def test_relay_failure_names_the_relay(monkeypatch, stdout, expected): from verifiers.v1.errors import RelayError diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 29fb52008e..8ce2af2967 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -140,7 +140,8 @@ async def failure(self) -> RelayError | None: ) with contextlib.suppress(Exception): result = await self.runtime.run(["sh", "-c", check], {}) - reasons = result.stdout.strip().splitlines() + # "ok": a request went through after the last failure. + reasons = [r for r in result.stdout.strip().splitlines() if r != "ok"] if reasons: return RelayError("; ".join(reasons)) return None @@ -151,8 +152,7 @@ async def stop(self) -> None: if not self._launch.done(): self._launch.cancel() await asyncio.wait({self._launch}) - if self._launch.cancelled() or self._launch.exception() is not None: - return + # Also after a failed or cancelled launch: the process may run already. pid, directory = shlex.quote(self.pid_path), shlex.quote(self.dir) # Best effort: the box may be gone already. with contextlib.suppress(Exception): diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index dfbe58bfdf..6c7561d4d9 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -175,6 +175,8 @@ def __init__(self, path: str, failures: str | None = None) -> None: self.path = path self.failures = failures self.failures_lock = threading.Lock() + self.failed = False + """Whether the last request outcome recorded in `failures` is a failure.""" self.config = Config.load(path) self.changed = threading.Condition() self.upstreams: set[Upstream] = set() @@ -188,6 +190,15 @@ def begin(self) -> tuple[Upstream, Config]: return upstream, self.config def record_failure(self, reason: str) -> None: + self._record(reason.replace("\n", " "), failed=True) + + def record_success(self) -> None: + """After a failure, record that a request went through again: the failure is + no longer the relay's current state.""" + if self.failed: + self._record("ok", failed=False) + + def _record(self, line: str, *, failed: bool) -> None: if self.failures is None: return with ( @@ -195,7 +206,8 @@ def record_failure(self, reason: str) -> None: contextlib.suppress(OSError), open(self.failures, "a") as file, ): - file.write(reason.replace("\n", " ") + "\n") + file.write(line + "\n") + self.failed = failed def end(self, upstream: Upstream) -> None: with self.changed: @@ -306,6 +318,11 @@ def relay(self) -> None: except ValueError: self.retry_count = 0 self.retries = 0 + self.sent = 0 + """Attempts that may have reached the interception server: the retry count it + sees. An attempt that failed to connect or got the tunnel edge's own page did + not, so a request that a legitimate later turn repeats is never taken for a + retry and answered from the server's replay cache.""" self.failing_since: float | None = None self.delay = BACKOFF_SECONDS self.sent_status: int | None = None @@ -323,12 +340,14 @@ def relay(self) -> None: log(f"{self.command} {self.label}: {error}") self.fail(str(error)) return + opened = time.monotonic() try: done = self.stream(upstream, response) finally: relay.end(upstream) connection.close() if done: + relay.record_success() retried = f" after {self.retries} retries" if self.retries else "" log(f"{self.command} {self.label} {self.sent_status}{retried}") return @@ -340,6 +359,9 @@ def relay(self) -> None: self.retries += 1 log(f"{self.command} {self.label}: aborted, retrying") else: + if time.monotonic() - opened > BACKOFF_CAP_SECONDS: + # The response was up a while: this is a new outage. + self.failing_since = None try: self.unreachable(relay.config, "the response broke") except UpstreamError as error: @@ -373,11 +395,11 @@ def upstream_headers(self, config: Config) -> dict[str, str]: lower = name.lower() if lower in ("authorization", "x-api-key"): value = config.swap(lower, value) - elif lower == RETRY_COUNT_HEADER and self.retries: + elif lower == RETRY_COUNT_HEADER and self.sent: continue headers[name] = f"{headers[name]}, {value}" if name in headers else value - if self.retries: - headers[RETRY_COUNT_HEADER] = str(self.retry_count + self.retries) + if self.sent: + headers[RETRY_COUNT_HEADER] = str(self.retry_count + self.sent) return headers def open( @@ -400,6 +422,7 @@ def open( self.body or None, self.upstream_headers(config), ) + self.sent += 1 response = connection.getresponse() except (OSError, http.client.HTTPException) as error: relay.end(upstream) @@ -411,8 +434,8 @@ def open( self.unreachable(config, f"{error!r}") continue if is_edge_error(response): - with contextlib.suppress(OSError, http.client.HTTPException): - response.read() + # Not read: an edge that stalls its page must not block the retry. + self.sent -= 1 relay.end(upstream) connection.close() self.unreachable(config, f"tunnel edge answered {response.status}") @@ -526,7 +549,11 @@ def start(self, response: http.client.HTTPResponse) -> None: if bodyless: if length := response.getheader("Content-Length"): self.send_header("Content-Length", length) - elif response.length is not None and not response.chunked: + elif ( + response.length is not None + and not response.chunked + and not self.scanner.sse + ): self.sent_length = response.length self.send_header("Content-Length", str(response.length)) elif self.request_version != "HTTP/1.0": diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index c5a4d9d49a..16346aaa78 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -391,7 +391,7 @@ async def open(self) -> bool: self._shared_tools, ) ) - harness_base = runtime.host_url(base_url.rstrip("/")) + harness_base = runtime.host_url(base_url.rstrip("/")).rstrip("/") if self._relay is not None: # The harness holds the model secret anyway; the relay gets # nothing more, so reading its config grants no capability. @@ -582,6 +582,10 @@ async def _reroute(self, base_url: str) -> None: """Move the harness's relay to the interception server's new URL (`RolloutSession.reroute`).""" assert self._relay is not None + if self._closed: + # Closing: the agent is done and the relay unused, and scoring keeps its + # own egress policy. + return await self._relay.wait_connected() upstream = base_url.rstrip("/") # A restricted box must reach the new tunnel host before the relay uses it. From 31d658b397b93653a711cacfd230c481a0f56ede Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 01:35:24 +0000 Subject: [PATCH 10/53] name RelayError in the relay docs Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 6984096ece..c685ad35b9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,7 +32,7 @@ A Prime tunnel can drop for a while (its client reconnects) or for good (the tun - While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. - When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. -A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a Python script that verifiers runs with uv; setup installs uv and a Python when the image has none. A relay that does not start fails the rollout with a `SandboxError`. +A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a uv script, like the harness programs and tool servers verifiers already runs in the box. A relay that does not start, gives up on a request or exits fails the rollout with a `RelayError`. ## Finishing a rollout From 01f4f571067f86645030f694b7faaf4ddbc3cc16 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 4 Nov 2026 23:51:53 +0000 Subject: [PATCH 11/53] feat: restore rollouts from filesystem checkpoints An agent snapshots its prime box as model turns go upstream, and a rollout retry restores the newest snapshot in a new box and continues the trace from there instead of starting over. On by default (`[env.agent.checkpoint]`: interval, keep, after, boxless). The trace records each snapshot (`Trace.checkpoints`), rewinds to the one a retry restores (`Trace.rewind`, `Trace.generation`), and gives another process what it needs to continue it (`Trace.restorable`, `RestorePoint`, `Env.run_slot(restore=...)`). null and bash relaunch on the transcript; claude-code, codex and rlm load their ACP session from the restored box (rlm pinned to nano-rlm with session/load). A request hook that raises now ends the rollout with its error. This is the filesystem-only state of verifiers#2994 as of ebfd9f8d2, on today's main. Its env-server parts moved with the env server to prime-rl and are not part of this change. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 6 + docs/harnesses.md | 1 + tests/unit/test_agent.py | 34 ++ tests/unit/test_checkpoint.py | 315 +++++++++++++ tests/unit/test_retry_ids.py | 4 + verifiers/v1/__init__.py | 3 + verifiers/v1/acp/__init__.py | 35 +- verifiers/v1/acp/runner.py | 44 +- verifiers/v1/agent.py | 230 ++++++++-- verifiers/v1/configs/agent.py | 42 +- verifiers/v1/env.py | 21 +- verifiers/v1/harness.py | 36 +- verifiers/v1/harnesses/bash/harness.py | 1 + verifiers/v1/harnesses/claude_code/harness.py | 9 +- verifiers/v1/harnesses/codex/harness.py | 4 +- verifiers/v1/harnesses/null/harness.py | 1 + verifiers/v1/harnesses/rlm/harness.py | 18 +- verifiers/v1/interception/server.py | 46 +- verifiers/v1/rollout.py | 428 +++++++++++++++--- verifiers/v1/runtimes/backends/prime.py | 64 ++- verifiers/v1/runtimes/base.py | 27 ++ verifiers/v1/session.py | 33 +- verifiers/v1/trace.py | 194 +++++++- 23 files changed, 1461 insertions(+), 135 deletions(-) create mode 100644 tests/unit/test_checkpoint.py diff --git a/docs/agent.md b/docs/agent.md index db057cc754..7ef3334272 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -72,3 +72,9 @@ max_turn_tokens = 4096 skip_on_truncated = true skip_on_timeout = true ``` + +## Retries and checkpoints + +`[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. + +Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box as model turns go upstream, and a retry restores the newest snapshot and continues the trace from there. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_CHECKPOINTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `interval` sets how often it snapshots, `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. diff --git a/docs/harnesses.md b/docs/harnesses.md index 54b1f418a1..e634a3e746 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,6 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). +- `SUPPORTS_FILESYSTEM_CHECKPOINTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem checkpoint, so a retry restores the newest checkpoint instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..3e4708edcd 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -18,6 +18,7 @@ class FailingSegmentRollout: closed = Rollout.closed fail = Rollout.fail step = Rollout.step + _turn = Rollout._turn timeout = Rollout.timeout @@ -39,6 +40,7 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._opened = True run._closed = False run._failed = False + run._attempt = 0 run._failure = None run._borrowed_runtime = None run.runtime = None @@ -49,6 +51,8 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._session = SimpleNamespace( request_interceptors=[], error=None, + aborted=None, + abort_event=asyncio.Event(), stopped=False, ) run.deadline_at = None @@ -99,3 +103,33 @@ async def turn(self, messages): assert trace.is_timeout and not trace.errors assert trace.stop_condition == "agent_timeout" + + +@pytest.mark.parametrize("outcome", ["keeps waiting", "dies"]) +async def test_request_hook_failure_fails_the_rollout(outcome): + """A request hook's failure is the task's verdict: it ends the segment at once and + fails the rollout, whatever the harness does meanwhile.""" + trace = make_trace() + error = vf.ProviderError("hook refused the turn", status_code=503) + cancelled = [] + + class WaitingSession: + async def turn(self, messages): + run._session.aborted = error + run._session.abort_event.set() + try: + if outcome == "keeps waiting": + await asyncio.sleep(60) + else: + raise RuntimeError("harness died") + except asyncio.CancelledError: + cancelled.append(True) + raise + + run = make_rollout(trace, WaitingSession(), time_remaining=5) + + await asyncio.wait_for(Interaction(run).turn("next"), timeout=1) + + assert [(e.type, e.stage) for e in trace.errors] == [("ProviderError", "agent")] + assert not trace.is_timeout + assert cancelled == ([True] if outcome == "keeps waiting" else []) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py new file mode 100644 index 0000000000..19b1918330 --- /dev/null +++ b/tests/unit/test_checkpoint.py @@ -0,0 +1,315 @@ +"""Rollout checkpointing: when a rollout takes checkpoints, what `rewind` restores, and +what a restored attempt must not redo.""" + +import asyncio +from types import SimpleNamespace +from typing import Literal + +import verifiers.v1 as vf +import verifiers.v1.agent as agent_module +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import RolloutCheckpointConfig +from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.graph import MessageNode +from verifiers.v1.rollout import Rollout, kept +from verifiers.v1.session import RolloutSession +from verifiers.v1.trace import ( + Checkpoint, + ModelCall, + RestorePoint, + TimeSpan, + TraceTask, +) +from verifiers.v1.types import ( + AssistantMessage, + Request, + ToolCall, + ToolMessage, + UserMessage, +) + + +def make_trace() -> vf.Trace: + trace = vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=TraceTask(type="TaskData", data=vf.TaskData(idx=0, prompt="q")), + ) + trace.nodes.append(MessageNode(parent=None, message=UserMessage(content="q"))) + trace.nodes.append( + MessageNode( + parent=0, + message=AssistantMessage( + content="", + tool_calls=[ToolCall(id="c1", name="bash", arguments="{}")], + ), + sampled=True, + ) + ) + trace.calls.append(ModelCall(node=1, time=TimeSpan(start=10.0, end=40.0))) + return trace + + +def test_rewind_restores_info_with_state(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", node=1, info={"setup": "nonce"}) + trace.checkpoints.append(checkpoint) + trace.info = {"setup": "nonce", "scoring_skipped": "stale"} + trace.rewind(checkpoint) + assert trace.info == {"setup": "nonce"} + + +def test_rewind_keeps_the_dropped_turns_calls(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", node=0) + trace.checkpoints.append(checkpoint) + trace.rewind(checkpoint) + assert len(trace.nodes) == 1 + assert [call.node for call in trace.calls] == [None] + + +def test_restored_span_counts_only_its_own_calls(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.split_agent_time() + assert trace.timing.agent.model.duration == 5.0 + assert trace.timing.agent.harness.duration == 15.0 + + +def test_agent_time_splits_off_the_checkpoint_hold(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.checkpoints += [ + Checkpoint(id="old", captured=90.0, held=3.0), + Checkpoint(id="new", captured=106.0, held=1.5), + ] + trace.split_agent_time() + assert trace.timing.agent.checkpoint.duration == 1.5 + assert trace.timing.agent.model.duration == 3.5 + assert trace.timing.agent.harness.duration == 15.0 + + +async def test_restored_tail_is_not_intercepted_twice(): + trace = make_trace() + seen: list[str] = [] + + def tag(request: Request) -> Request: + for message in request.messages: + if isinstance(message, ToolMessage | UserMessage): + seen.append(str(message.content)) + if isinstance(message, ToolMessage): + message.content = f"{message.content} [tagged]" + return request + + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), + trace=trace, + request_interceptors=[tag], + ) + tail = [ + ToolMessage(tool_call_id="c1", content="out [tagged]"), + UserMessage(content="go on"), + ] + session.mark_intercepted(trace.messages, tail) + request = Request(messages=[*trace.messages, *tail]) + rewritten, records, _ = await session.rewrite_request(request) + assert rewritten.messages == request.messages + assert records == [] + assert seen == [] + + +class SlowBackend: + """Captures at once; becomes durable when the test says so.""" + + def __init__(self) -> None: + self.durable = asyncio.Event() + self.fail = False + self.taken = 0 + self.deleted: list[str] = [] + + async def checkpoint(self) -> str: + self.taken += 1 + return f"ck{self.taken}" + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + await self.durable.wait() + if self.fail: + raise RuntimeError("upload failed") + return 1024 + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + self.deleted += ids + return ids + + +def make_rollout( + interval: int = 1, keep: Literal["all", "last"] | int = "last" +) -> tuple[Rollout, SlowBackend]: + backend = SlowBackend() + run = Rollout.__new__(Rollout) + run.trace = make_trace() # one turn so far + run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] + + async def checkpoint_state() -> dict: + return {} + + run._harness_session = SimpleNamespace(checkpoint_state=checkpoint_state) # type: ignore[assignment] + run._session = SimpleNamespace(pinned_tool_results={}) # type: ignore[assignment] + run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) + run._durable = {} + run._captures = set() + run._closed = False + run._pruning = asyncio.Lock() + run.deleted_checkpoints = set() + run.rewound_checkpoints = [] + run._boxless = False + return run, backend + + +TURN = SimpleNamespace(prefix_node_ids=[0, 1], tail=[]) + + +async def test_checkpoint_interval_skips_turns(): + run, backend = make_rollout(interval=2) + assert await run._checkpoint(TURN) is None # turn 2: 1 % 2 + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + run.trace.nodes.append( + MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) + ) + assert await run._checkpoint(TURN) is not None # turn 3: 2 % 2 + assert backend.taken == 1 + + +async def test_checkpoint_becomes_durable_in_the_background(): + run, backend = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + assert checkpoint is not None and checkpoint.captured + assert checkpoint.time.end == 0 and checkpoint.bytes is None + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert checkpoint.time.end and checkpoint.bytes == 1024 + assert run.trace.checkpoints == [checkpoint] + + +async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): + run, backend = make_rollout(interval=1) + await run._checkpoint(TURN) + backend.fail = True + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert run.trace.checkpoints == [] + + +async def test_settling_without_a_retry_stops_waiting(): + run, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + await asyncio.wait_for(run._settle_checkpoints(wait=False), timeout=1) + assert run._durable == {} + assert run.trace.checkpoints == [checkpoint] and checkpoint.time.end == 0 + + +async def test_a_closing_run_takes_no_checkpoint(): + run, backend = make_rollout(interval=1) + capture = asyncio.create_task(run._checkpoint(TURN)) + run._closed = True + await run._settle_checkpoints(wait=True) + assert await capture is None + assert backend.taken == 0 and run.trace.checkpoints == [] + + +def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: + return [ + Checkpoint(id=i, time=TimeSpan(start=1.0, end=2.0), deleted=i in deleted) + for i in ids + ] + + +def test_keep_policies(): + checkpoints = durable("a", "b", "c", "d", "e") + assert kept(checkpoints, "last") == {"e"} + assert kept(checkpoints, "all") == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 2) == {"a", "c", "e"} + assert kept(checkpoints, 3) == {"a", "d", "e"} + # Deleted and not yet durable snapshots are never kept. + pending = [*durable("a", "b", deleted=("a",)), Checkpoint(id="c")] + assert kept(pending, "all") == {"b"} + + +async def test_a_durable_snapshot_prunes_the_older_ones(): + run, backend = make_rollout(keep="last") + run.trace.checkpoints = durable("old1", "old2") + run.rewound_checkpoints = ["rewound"] + backend.durable.set() + newest = await run._checkpoint(TURN) + await run._settle_checkpoints(wait=True) + assert sorted(backend.deleted) == ["old1", "old2", "rewound"] + assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] + assert run.deleted_checkpoints == {"old1", "old2", "rewound"} + + +async def test_a_boxless_checkpoint_is_restorable_at_once(): + run, backend = make_rollout() + run._boxless = True + checkpoint = await run._checkpoint(TURN) + assert checkpoint is not None and not checkpoint.box + assert checkpoint.time.end and backend.taken == 0 + assert run.trace.restorable is checkpoint + + +def test_a_restore_point_round_trips(): + checkpoint = Checkpoint( + id="ck", + node=1, + tail=[ToolMessage(tool_call_id="c1", content="out")], + state=vf.State(), + info={"nonce": "n"}, + pins={("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned")}, + time=TimeSpan(start=1.0, end=2.0), + ) + wire = RestorePoint.of(checkpoint).model_dump(mode="json") + back = RestorePoint.model_validate(wire).live(vf.State) + assert back.tail == checkpoint.tail + assert back.info == {"nonce": "n"} + assert back.pins == checkpoint.pins + assert back.state == checkpoint.state + + +def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: + return vf.Agent( + vf.AgentConfig( + model="test-model", + harness={"id": harness}, + runtime=runtime, + checkpoint=RolloutCheckpointConfig(**checkpoint), + ) + ) + + +def test_checkpointing_is_on_by_default(): + assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() + assert gated("bash", vf.PrimeConfig())._checkpointing() is not None + + +def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): + monkeypatch.setattr(agent_module, "_checkpointing_warned", set()) + for _ in range(2): + assert gated("bash", vf.SubprocessConfig())._checkpointing() is None + assert gated("terminus-2", vf.PrimeConfig())._checkpointing() is None + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert warnings == [ + ( + "rollout checkpointing is off: the 'subprocess' runtime takes no " + "snapshots; a failed rollout retries from the start" + ), + ( + "rollout checkpointing is off: harness 'terminus-2' does not support it; " + "a failed rollout retries from the start" + ), + ] + + +def test_boxless_checkpointing_is_opt_in(): + assert gated("null", vf.SubprocessConfig())._checkpointing() is None + assert gated("null", vf.SubprocessConfig(), boxless=True)._checkpointing() + assert gated("bash", vf.SubprocessConfig(), boxless=True)._checkpointing() is None diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 80cb35978e..7c3d6f9613 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -33,7 +33,11 @@ async def test_agent_retries_keep_the_trace_id(monkeypatch): minted: list[vf.Trace] = [] class FakeRollout: + runtime = None + def __init__(self, *, on_trace, trace_id, attempt, **_): + self.rewound_checkpoints: list[str] = [] + self.deleted_checkpoints: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index ecaf8b4d6e..33ff9dd52b 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -119,6 +119,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, EvalRunInfo, EvalWorkInfo, @@ -209,6 +210,7 @@ "BaseConfig": "pydantic_config", "BindMount": "verifiers.v1.configs.runtime", "Branch": "verifiers.v1.trace", + "Checkpoint": "verifiers.v1.trace", "Client": "verifiers.v1.clients", "ClientConfig": "verifiers.v1.configs.client", "ComposeSpec": "verifiers.v1.runtimes.compose", @@ -430,6 +432,7 @@ "StateT", "MessageNode", "Branch", + "Checkpoint", "TurnTokens", "SamplingMask", "TopLogprobs", diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 4bed677ef5..d5fec9315b 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -14,7 +14,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.errors import HarnessError, InterceptionError -from verifiers.v1.harness import Harness, HarnessSession +from verifiers.v1.harness import RESTORE_PROMPT, Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess from verifiers.v1.semantic import ( ACP_SEMANTIC_EDGES_METADATA_KEY, @@ -22,7 +22,7 @@ ) from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace -from verifiers.v1.types import Messages +from verifiers.v1.types import Messages, UserMessage from verifiers.v1.utils.aio import run_shielded ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() @@ -237,6 +237,10 @@ def __init__( self._responses: dict[int, asyncio.Future] = {} self._request_id = 0 self._response_task: asyncio.Task | None = None + self._session_id: str | None = None + """The agent's ACP session id, once a checkpoint asked for it.""" + self._restore_id: str | None = None + """The session the next process loads from the restored box (`restore`).""" async def _start(self) -> None: self._stderr_tail.clear() @@ -284,6 +288,28 @@ async def _request(self, operation: str, **payload: Any) -> JsonObject: if not future.done(): future.cancel() + async def checkpoint_state(self) -> dict[str, Any] | None: + # The agent keeps its session in files of the box, so a restored box can load + # it in a new process — if the agent advertises `loadSession`. + if self._session_id is None: + if self._process is None: + return None + response = await self._request("session") + result = response.get("result") + if not response.get("ok") or not isinstance(result, dict): + raise RuntimeError(response.get("error") or "ACP session query failed") + if not result.get("load_session") or result.get("session_id") is None: + return None + self._session_id = result["session_id"] + return {"acp_session_id": self._session_id} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + # The agent's own session files hold the conversation, the tail included as + # far as it flushed them; the restored process takes only the notice. + self._restore_id = state["acp_session_id"] + self._session_id = self._restore_id + return [UserMessage(content=RESTORE_PROMPT)] + async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed: raise RuntimeError("ACP session is closed") @@ -326,6 +352,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult: ) config = { "command": self.config.command, + "load_session_id": self._restore_id, "user_contents": user_contents, "mcp_urls": self.mcp_urls, "mcp_headers": self.harness.config.resolve_mcp_headers(self.mcp_urls), @@ -354,6 +381,10 @@ async def _run(self, messages: Messages | None) -> ProgramResult: except BaseException: await run_shielded(self._stop(graceful=False)) raise + finally: + # Only the restored box's first process loads the session; later + # turns continue the live one. + self._restore_id = None turn = _turn_result(response) self.trace.root_reply = turn.reply.strip() self.trace.info["acp_stop_reason"] = turn.stop_reason diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index b19942b5b9..5a7ec185af 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -87,6 +87,8 @@ def __init__(self) -> None: self.stop_reason: str | None = None self.response_metadata: dict[str, Any] = {} self.update_metadata: list[dict[str, Any]] = [] + self.loading = False + """Set while `session/load` replays the history, which no turn reports.""" def reset(self) -> None: self.visible_reply = "" @@ -104,6 +106,8 @@ def turn_result(self) -> ACPTurn: ) async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: + if self.loading: + return metadata = dict(kwargs) if isinstance(field_meta := getattr(update, "field_meta", None), dict): metadata.update(field_meta) @@ -267,18 +271,37 @@ async def start(self, config: dict) -> None: self.supports_steering = (initialized.field_meta or {}).get( "steering", {} ).get("supported") is True - session = await self.connection.new_session( - cwd=os.getcwd(), - mcp_servers=mcp_servers(config), - **config["session_meta"], - ) + load_id = config.get("load_session_id") + if load_id is None: + session = await self.connection.new_session( + cwd=os.getcwd(), + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + session_id = session.session_id + else: + # A box restored from a checkpoint: the agent's session files are + # here, its process is not. Load the session in this new process. + if not (self.capabilities and self.capabilities.load_session): + raise RuntimeError("ACP agent does not support session/load") + self.client.loading = True + try: + await self.connection.load_session( + cwd=os.getcwd(), + session_id=load_id, + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + finally: + self.client.loading = False + session_id = load_id except BaseException: with suppress(BaseException): await self.stack.aclose() self._reset() raise - self.session_id = session.session_id - self.is_new = True + self.session_id = session_id + self.is_new = load_id is None async def run(self, config: dict) -> ACPTurn: gate = config.get("tool_interception") @@ -381,6 +404,13 @@ async def handle(request: dict) -> None: result = await session.steer( request["message"], request.get("message_id") ) + elif operation == "session": + result = { + "session_id": session.session_id, + "load_session": bool( + session.capabilities and session.capabilities.load_session + ), + } elif operation == "shutdown": result = {"response_metadata": await session.close()} else: diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index ba94921c36..449ce0718d 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -25,6 +25,7 @@ ) from verifiers.v1.configs.agent import ( AgentConfig, + RolloutCheckpointConfig, TimeoutConfig, agent_config_fields, resolve_agent, @@ -34,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.rollout import Rollout, RolloutTimeouts +from verifiers.v1.rollout import Rollout, RolloutTimeouts, kept from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -44,7 +45,7 @@ ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -63,6 +64,9 @@ logger = logging.getLogger(__name__) +# (harness id, runtime type) pairs already warned that they take no checkpoints. +_checkpointing_warned: set[tuple[str, str]] = set() + # Only nested runs in this scoring attempt participate in its rollback. _scoring_runs: ContextVar[list[tuple["_EpisodeAgent", Trace]] | None] = ContextVar( "scoring_runs", default=None @@ -571,6 +575,9 @@ def __init__( sampling=config.sampling, ) self._closed = False + self.pending_restore: tuple[Trace, Checkpoint] | None = None + """A restore the next `run` starts from (a trace another process ran); consumed + by that run.""" self.runtime_config: RuntimeConfig = config.runtime self.interception = interception self.limits = RolloutLimits( @@ -641,6 +648,34 @@ def _interception_for( return self._server return None + def _checkpointing(self) -> RolloutCheckpointConfig | None: + """The checkpoint config when this harness on this runtime can take and restore + checkpoints; otherwise None, with a warning once per harness and runtime.""" + config = self.config.checkpoint + if config is None: + return None + harness = self.harness + backend = backend_cls(self.config.runtime) + if harness.SUPPORTS_FILESYSTEM_CHECKPOINTS and ( + backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + or (config.boxless and not harness.EXECUTES_CODE) + ): + return config + key = (harness.config.id, self.config.runtime.type) + if key not in _checkpointing_warned: + _checkpointing_warned.add(key) + reason = ( + f"harness {harness.config.id!r} does not support it" + if not harness.SUPPORTS_FILESYSTEM_CHECKPOINTS + else f"the {self.config.runtime.type!r} runtime takes no snapshots" + ) + logger.warning( + "rollout checkpointing is off: %s; a failed rollout retries from the " + "start", + reason, + ) + return None + def _check_resume_support(self) -> None: # Multi-turn capability is derived: a harness needs transcript replay, a # native resume implementation, or a rollout-scoped session implementation. @@ -667,6 +702,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -675,49 +711,149 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; every attempt's trace keeps - the run's id with its own `attempt`, and the final trace keeps earlier - attempts' errors. `on_score(trace, runtime)` runs after task scoring in + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry restores the failed attempt's newest + restorable checkpoint when it has one (`config.checkpoint`, on a runtime and a + harness that support it), keeping that trace and its turns up to there; a + checkpoint whose restore failed is skipped, and without one the retry starts + over. Every attempt's trace keeps the run's id with its own `attempt`, and the + final trace keeps every attempt's errors. `on_score(trace, runtime)` runs after task scoring in the selected verifier workspace, inside its timeout and retry boundary.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") + if restore is None and self.pending_restore is not None: + restore, self.pending_restore = self.pending_restore, None retry = self.config.retries retry_state = RetryState(retry) - history: list = [] - trace_id = uuid.uuid4().hex - for attempt in range(retry_state.max_retries + 1): - trace = await self._run_once( - task, - runtime, - tools, - on_trace, - collect_artifacts, - on_score, - trace_id=trace_id, - attempt=attempt + 1, - ) - if attempt == retry_state.max_retries or trace.ok: - break - if runtime is not None: + checkpointing = self._checkpointing() if runtime is None else None + # One id for the run across its attempts; a restored trace keeps its own. + trace_id = restore[0].id if restore is not None else uuid.uuid4().hex + history: list[Error] = [] # errors of the attempts started over from + # `restore` continues a trace another process ran (an orchestrator resuming its + # run): its first attempt restores it, and `on_trace` sees it as the run's trace. + trace: Trace | None = restore[0] if restore is not None else None + target: Checkpoint | None = restore[1] if restore is not None else None + if restore is not None and on_trace is not None: + on_trace(restore[0]) + snapshots: set[str] = set() # every checkpoint any attempt took + deleted: set[str] = set() + failed_restores: set[str] = set() + run: Rollout | None = None + # The `finally` also covers a cancellation between attempts, when no rollout is + # open to delete them on abort. + budget = retry_state.max_retries + attempt = 0 + try: + while True: + restoring = ( + (trace, target) + if trace is not None and target is not None + else None + ) + # This attempt's own errors decide — a restored trace still carries + # the earlier attempts'. + seen = len(restoring[0].errors) if restoring is not None else 0 + run = await self._run_once( + task, + runtime, + tools, + on_trace, + collect_artifacts, + on_score, + trace_id=trace_id, + attempt=attempt + 1, + restore=restoring, + checkpoint=checkpointing, + ) + trace = run.trace + snapshots.update(run.rewound_checkpoints) + snapshots.update(c.id for c in trace.checkpoints if c.box) + deleted.update(run.deleted_checkpoints) + if target is not None and any( + error.stage == "setup" for error in trace.errors[seen:] + ): + # The box never booted from it: the next retry steps back. + failed_restores.add(target.id) + if trace.ok: + break + if ( + attempt == 0 + and restore is not None + and target is not None + and target.id in failed_restores + ): + # Another process's restore that cannot boot (its snapshot is gone) + # is not the rollout's failure: start over, unbudgeted. + logger.warning( + "restoring trace %s from checkpoint %s failed; starting over", + trace.id, + target.id, + ) + history.extend(trace.errors) + target = None + trace = None + attempt += 1 + budget += 1 + continue + if attempt >= budget: + break + if runtime is not None: + logger.warning( + "not retrying the rollout on a borrowed box (its state is no " + "longer the task's start state); the error stands" + ) + break + cause = retry_state.next_error(trace.errors[seen:]) + if cause is None: + break + target = next( + ( + c + for c in reversed(trace.checkpoints) + if c.time.end and not c.deleted and c.id not in failed_restores + ), + None, + ) + if target is None: + history.extend(trace.errors) + delay = backoff(attempt) logger.warning( - "not retrying the rollout on a borrowed box (its state is no " - "longer the task's start state); the error stands" + "retrying agent rollout (retry %d/%d) in %.1fs after error: %s (%s)", + attempt + 1, + budget, + delay, + cause.type, + f"restoring checkpoint {target.id}" + if target is not None + else "from scratch", ) - break - cause = retry_state.next_error(trace.errors) - if cause is None: - break - history.extend(trace.errors) - delay = backoff(attempt) - logger.warning( - "retrying agent rollout (retry %d/%d) in %.1fs after error: %s", - attempt + 1, - retry_state.max_retries, - delay, - cause.type, + await asyncio.sleep(delay) + attempt += 1 + finally: + # The run is final (or cancelled): delete what `after` does not keep. Every + # box is gone by now, so the platform refuses none of them. + keep = ( + kept(run.trace.checkpoints, checkpointing.keep) + if run is not None + and checkpointing is not None + and checkpointing.after == "keep" + else set() ) - await asyncio.sleep(delay) + drop = sorted(snapshots - deleted - keep) + if drop and run is not None and run.runtime is not None: + try: + run.record_deleted( + await run_shielded( + run.runtime.backend.discard_checkpoints(drop) + ) + ) + except Exception: + logger.warning( + "discarding checkpoints failed (rollout %s)", + run.trace.id, + exc_info=True, + ) + assert trace is not None and run is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -735,7 +871,9 @@ async def _run_once( *, trace_id: str, attempt: int, - ) -> Trace: + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, + ) -> Rollout: params = self._rollout_params(task, runtime, dict(shared_tools or {})) if ( collect_artifacts @@ -751,12 +889,17 @@ async def _run_once( collect_artifacts=collect_artifacts, on_score=on_score, trace_id=trace_id, + restore=restore, + checkpoint=checkpoint, attempt=attempt, **params, ) try: if await run.open(): - await run.step() + if restore is not None: + await run.restore() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() @@ -766,7 +909,7 @@ async def _run_once( raise if trace.agent.runtime is not None: trace.agent.runtime.borrowed = runtime is not None - return trace + return run @asynccontextmanager async def interaction( @@ -995,6 +1138,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: async with self._gate or nullcontext(): trace = await super().run( @@ -1004,6 +1148,7 @@ async def run( on_trace=self._watch(on_trace), collect_artifacts=collect_artifacts, on_score=self._scoring_callback(on_score), + restore=restore, ) self._completed.append(trace) return trace @@ -1071,6 +1216,11 @@ def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None: for name, value in agent_config_fields(config).items() } + def only(self) -> AgentT | None: + """The agent of a config with one agent seat; None when it has several.""" + agents = list(self._agents.values()) + return agents[0] if len(agents) == 1 else None + def __getattr__(self, name: str) -> AgentT: # self.__dict__ directly: attribute lookup re-entering __getattr__ before # __init__ ran (copy/unpickle) must raise, not recurse. diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 19d484563d..c9aa905980 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -1,10 +1,11 @@ """One env agent's config: who plays the seat, and its per-run caps.""" -from typing import Any +from typing import Any, Literal from pydantic import ( BaseModel, Field, + PositiveInt, SerializationInfo, SerializeAsAny, field_serializer, @@ -38,6 +39,39 @@ class TimeoutConfig(BaseConfig): isolated verifier setup and retries. Finalization has its own budget.""" +class RolloutCheckpointConfig(BaseConfig): + """Rollout checkpointing. Snapshot the rollout's box (its filesystem) as model + turns go upstream, so a retry restores the newest snapshot instead of starting + over: the trace keeps its turns up to there, and a new harness process continues + in a box restored to that state, told that the box was restored. Needs a runtime + with checkpoints (prime) and a harness that can restore one (both set + `SUPPORTS_FILESYSTEM_CHECKPOINTS`); elsewhere the agent warns once and a retry starts over. + The snapshot runs alongside the model call, and the reply waits only until its + content is fixed. + + Known limits: a harness with concurrent sub-agents can be snapshotted while one of + them is mid-tool, and processes the agent left running (background jobs, + sub-agents, a REPL's variables) are gone after a restore, which cold boots.""" + + interval: PositiveInt = 1 + """Snapshot before every this-many-th model turn (the 1st, the 1+n-th, ...). A + larger interval takes fewer snapshots, and a restore loses up to this many turns.""" + keep: Literal["all", "last"] | PositiveInt = "last" + """Which snapshots stay while the rollout runs: `"last"` keeps only the newest + restorable one, `"all"` keeps every one, and `k` keeps every k-th (the 1st, the + 1+k-th, ...) plus the newest. The others are deleted as soon as a newer one is + restorable. A retry whose restore fails falls back to an older snapshot that is + still kept, else it starts over.""" + after: Literal["delete", "keep"] = "delete" + """What happens to the kept snapshots when the rollout's run is final (it succeeded, + or its retries ran out): `"delete"` deletes them once the box is gone; `"keep"` + leaves them for their owner (forks, inspection), who must delete them.""" + boxless: bool = False + """Also checkpoint a harness that runs no code (null) on a runtime without + snapshots, from the trace alone. Off by default: such a run records a checkpoint + on its trace at every snapshot turn.""" + + class AgentConfig(BaseConfig): harness: SerializeAsAny[HarnessConfig] | None = None """The agent's program (None = the taskset's default harness).""" @@ -75,6 +109,12 @@ class AgentConfig(BaseConfig): timeout: TimeoutConfig = TimeoutConfig() retries: RetryConfig = RetryConfig() + checkpoint: RolloutCheckpointConfig | None = Field( + default_factory=RolloutCheckpointConfig + ) + """Rollout checkpointing, on by default: snapshot the box as turns go upstream, so + a retry restores the newest snapshot instead of starting over (see + `RolloutCheckpointConfig`). None turns it off.""" network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1f943574fa..2223174e5c 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -32,7 +32,7 @@ from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task -from verifiers.v1.trace import Error, Trace, TraceTask +from verifiers.v1.trace import Checkpoint, Error, Trace, TraceTask from verifiers.v1.utils.generic import concrete_type from verifiers.v1.utils.memory import trim_memory_periodically from verifiers.v1.utils.retries import run_episode_with_retry @@ -244,6 +244,7 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -266,6 +267,17 @@ async def run_episode( ), ) agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard) + if restore is not None: + # A one-seat episode's run is its one trace: its agent's run continues it. + agent = agents.only() + if agent is None: + logger.warning( + "%s has several agent seats; running trace %s from scratch", + type(self).__name__, + restore[0].id, + ) + else: + agent.pending_restore = restore try: async with asyncio.timeout(self.config.timeout.episode): async with boundary(EnvError, f"{type(self).__name__}.setup()"): @@ -320,13 +332,17 @@ async def run_slot( semaphore: asyncio.Semaphore | None = None, on_complete: Callable[[Episode], Awaitable[None]] | None = None, on_trace: Callable[[Trace], None] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """Run one planned episode to completion, with whole-episode retries per `--env.retries`; `semaphore` bounds concurrent EPISODES — one permit for the attempt in flight, held across the whole of it (its agents, their boxes, `finalize()`) and released before a retry's backoff and before `on_complete` (the runners' persistence hook, which fires when final). - `on_trace` sees each trace at mint, after it joined `slot.traces`.""" + `on_trace` sees each trace at mint, after it joined `slot.traces`. `restore` + continues a trace another process ran (one-seat episodes); only the first + attempt restores it, a whole-episode retry starts over.""" + pending = [restore] if restore is not None else [] attempts = 0 @@ -354,6 +370,7 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, + restore=pending.pop() if pending else None, ) episode = await run_episode_with_retry(attempt, self.config.retries) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index a4329bbe61..963d63c931 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -5,7 +5,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import DialectName, HarnessConfig, skill_destination @@ -18,7 +18,7 @@ ) from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData -from verifiers.v1.types import Message, Messages +from verifiers.v1.types import Message, Messages, UserMessage from verifiers.v1.utils.decorators import ( discover_decorated, invoke_all, @@ -36,6 +36,16 @@ ConfigT = TypeVar("ConfigT", bound=HarnessConfig) +RESTORE_PROMPT = ( + "Your workspace was restored from a filesystem checkpoint taken before your last " + "model turn, and you are continuing in a new process. Files are as they were at " + "the checkpoint. Background processes, running shells, sub-agents, and in-memory " + "state such as REPL variables from before the restore are gone; start again " + "whatever you still need. Your last action may not have taken effect, so check " + "before you rely on it. Continue the task." +) +"""The user turn a harness gets after a restore (`HarnessSession.restore`).""" + class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False @@ -49,6 +59,13 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + """Whether a new process of this harness can continue a rollout in a box restored + from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the + newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout + checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_CHECKPOINTS` too. Distinct + from `SUPPORTS_RESUME`: a restore never finds a live process, its program + continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True """Whether the program hands the model local execution in the runtime — true for every real harness; the tool-less chat loops (`null`) override to False. Read @@ -404,6 +421,21 @@ async def _run(self, messages: Messages | None) -> ProgramResult: self.tool_interception_url, ) + async def checkpoint_state(self) -> dict[str, Any] | None: + """What `restore` needs to continue in a box restored from a checkpoint taken + now, or None when this session cannot be restored (no checkpoint is taken). + The default relaunches on the trace's transcript and needs nothing.""" + return {} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + """Prepare this new session to continue a rollout in a box restored from a + checkpoint, and return the messages of its first segment. `state` is what + `checkpoint_state` returned when the checkpoint was taken; `tail` is the + checkpoint's uncommitted prompt after the trace's branch. The default + relaunches on the branch plus `tail`, then tells the program about the + restore (`RESTORE_PROMPT`).""" + return [*tail, UserMessage(content=RESTORE_PROMPT)] + async def close(self) -> None: """Close session-owned resources. Idempotent.""" self._closed = True diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 8f6d852144..e2790dfbd4 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,6 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 3287be9994..e2b4495e27 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -19,7 +19,7 @@ ACP_VERSION = "0.99.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" -CLAUDE_CONFIG_ROOT = ".vf-claude" +STATE_ROOT = "/tmp/vf-claude-code-runs" ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -42,6 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed @@ -104,6 +105,10 @@ async def prepare_acp( "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"), "ANTHROPIC_API_KEY": secret, "ANTHROPIC_MODEL": ctx.model, + # Claude Code validates a model it has not seen in this process with a + # one-token "Hi" request (e.g. when a restored session re-selects its + # model); a custom model option is trusted without that request. + "ANTHROPIC_CUSTOM_MODEL_OPTION": ctx.model, "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions), "CLAUDE_CONFIG_DIR": config_dir, "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", @@ -135,4 +140,4 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def config_dir(trace: Trace) -> str: - return f"{CLAUDE_CONFIG_ROOT}/{trace.id}" + return f"{STATE_ROOT}/{trace.id}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 59b690ac33..e45406ab40 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,6 +22,7 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" +STATE_ROOT = "/tmp/vf-codex-runs" ACP_VERSION = "2.5.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" @@ -62,6 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: @@ -174,7 +176,7 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def trace_home(trace: Trace) -> str: - return f"/tmp/vf-codex-home-{trace.id}" + return f"{STATE_ROOT}/{trace.id}" async def build_env( self, diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 503429ddb6..431c95f698 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,6 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 27d4136748..1c22e675e7 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -31,8 +31,8 @@ BuiltinTool = Literal["bash", "edit", "fetch", "ipython"] RLM_REPO = "github.com/PrimeIntellect-ai/nano-rlm.git" -RLM_CACHE_DIR = "/tmp/vf-rlm" -RLM_STATE_DIR = "/tmp/vf-rlm-state" +RLM_CACHE_DIR = "/var/tmp/vf-rlm" +STATE_ROOT = "/tmp/vf-rlm-runs" RLM_RUNTIME_METADATA_KEY = "ai.prime.rlm/runtime-v1" RLM_SESSION_METADATA_KEY = "ai.prime.rlm/session-v1" @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="ae1cc01f8293987d18b09286b34d93c0dfbf8dac", min_length=1 + default="a057f58d144384fed6ecfed1dabcf350fe540910", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" @@ -142,12 +142,18 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - self._skills_install_dir = f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + # A retry keeps the directory: it holds `RLM_HOME`, whose session a box + # restored from a checkpoint loads. + if self._skills_install_dir is None: + self._skills_install_dir = ( + f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + ) await self.install_skills(runtime, f"{self._skills_home()}/skills") directory = self._install_dir() binary = f"{directory}/bin/rlm" @@ -306,14 +312,14 @@ def acp_close_result(self, trace: Trace, response_metadata: dict[str, Any]) -> N self._consume_snapshot(trace, response_metadata) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - await runtime.run(["rm", "-rf", f"{RLM_STATE_DIR}/{trace.id}"], {}) + await runtime.run(["rm", "-rf", f"{STATE_ROOT}/{trace.id}"], {}) if self._skills_install_dir is not None: await remove_dir( runtime, self._skills_install_dir, "RLM skill installation" ) def _home(self, trace: Trace) -> str: - return self._skills_home() or f"{RLM_STATE_DIR}/{trace.id}/home" + return self._skills_home() or f"{STATE_ROOT}/{trace.id}/home" def _skills_home(self) -> str | None: """rlm's home for a run that carries skills: they sit in its `skills/`, where diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 8f8a0b3351..a82a177bc4 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -33,6 +33,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass, field from functools import partial +from typing import NoReturn import httpx from aiohttp import web @@ -606,10 +607,13 @@ async def start(self) -> None: ) def _fail( - self, session: RolloutSession, dialect: Dialect, error: RolloutError + self, + session: RolloutSession, + dialect: Dialect, + error: RolloutError, ) -> web.Response: - """Stash a model-turn-adjacent failure (such as a hook raising) so the rollout - re-raises it as the real cause, and report it to the harness as an HTTP error.""" + """Stash a model-turn-adjacent failure so the rollout re-raises it as the real + cause, and report it to the harness as an HTTP error.""" session.error = error logger.warning( "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error @@ -618,6 +622,17 @@ def _fail( dialect, str(error), status=getattr(error, "status_code", 502) ) + async def _abort(self, session: RolloutSession, error: RolloutError) -> NoReturn: + """End the rollout's segment with a request hook's failure. The harness gets no + reply: the rollout cancels its turn, and releasing the session cancels this + handler.""" + logger.warning( + "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error + ) + session.abort(error) + await asyncio.get_running_loop().create_future() + raise AssertionError("unreachable") + def mediate_capabilities( self, session: RolloutSession, dialect: Dialect, body: dict ) -> tuple[dict, list[str]]: @@ -870,11 +885,10 @@ async def coalesced( if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) except RolloutError as error: - return self._fail(session, dialect, error) + return await self._abort(session, error) except Exception as error: # noqa: BLE001 - surface task hook failures - return self._fail( + return await self._abort( session, - dialect, TaskError( f"model boundary hook failed: {type(error).__name__}: {error}" ), @@ -932,6 +946,14 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + # The box is idle while the harness awaits this turn, so a snapshot captured + # before the reply returns matches the prompt exactly (tool results in, this + # reply not yet). It runs alongside the model call; the reply waits for it. + capture = ( + asyncio.create_task(session.checkpointer(turn)) + if session.checkpointer is not None + else None + ) try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1024,6 +1046,18 @@ async def sample() -> web.Response: error = e raise finally: + if capture is not None: + held = time.monotonic() + try: + checkpoint = await capture + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", session.trace.id, exc_info=True + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held if node is None: turn.abandon() # The turn's one per-exchange record: settings, timing, outcome, and diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index a5563b982a..96ad61ea28 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -2,15 +2,19 @@ import asyncio import contextlib +import copy import logging import time +import uuid from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from typing import Literal from urllib.parse import urlsplit +from verifiers.v1 import graph from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.agent import AgentConfig +from verifiers.v1.configs.agent import AgentConfig, RolloutCheckpointConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.configs.scoring import resolve_scoring from verifiers.v1.errors import ( @@ -39,8 +43,8 @@ from verifiers.v1.runtimes.private_network import tool_vms from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary from verifiers.v1.state import state_cls -from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.task import Task, TaskData +from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke @@ -97,6 +101,25 @@ async def resolve_agent_runtime( return view +def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> set[str]: + """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every one for + `"all"` or every k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + live = [ + (index, c) + for index, c in enumerate(checkpoints) + if c.time.end and not c.deleted + ] + if not live: + return set() + if keep == "all": + return {c.id for _, c in live} + held = {live[-1][1].id} + if keep != "last": + held.update(c.id for index, c in live if index % keep == 0) + return held + + class Rollout: """Manages one rollout's lifecycle (open, step, close).""" @@ -119,7 +142,18 @@ def __init__( on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, trace_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, ) -> None: + """`attempt` numbers this run among the trace's retries, from 1 (stamped on + the trace and the errors it captures). `restore` continues a failed attempt's trace from one of + its checkpoints: the trace rewinds to it and this run boots its box from the + snapshot instead of setting the task up again. `checkpoint` snapshots the box as + model turns go upstream (on a runtime and a harness that support it) for a + retry to restore, and deletes the ones its `keep` policy drops as newer ones + become restorable; the owner of the retries deletes the rest once the run is + final (`RolloutCheckpointConfig.after`) — this rollout deletes them only when it + aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -139,23 +173,44 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - **({"id": trace_id} if trace_id is not None else {}), - attempt=attempt, - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) + self._checkpointing = checkpoint + self._durable: dict[str, asyncio.Task[None]] = {} + self._captures: set[asyncio.Task] = set() + """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" + """The snapshots still on their way to restorable, by checkpoint id.""" + self._attempt = attempt + self._restore: Checkpoint | None = None + self._harness_data: TaskData | None = None + self.deleted_checkpoints: set[str] = set() + """The snapshots this run deleted.""" + self._pruning = asyncio.Lock() + self._boxless = False + """Whether this run's checkpoints are box-less (`Checkpoint.box`).""" + self.rewound_checkpoints: list[str] = [] + """Checkpoints the restored trace dropped (taken after the one restored).""" + if restore is not None: + self.trace, self._restore = restore + self.trace.attempt = attempt + self.rewound_checkpoints = self.trace.rewind(self._restore) + self._restore.restored = True + else: + self.trace = Trace( + **({"id": trace_id} if trace_id is not None else {}), + attempt=attempt, + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) self.trace._replay_view = harness.replay_view - if on_trace is not None: - on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -217,10 +272,11 @@ def failure(self) -> Exception | None: """The original exception most recently captured onto the trace.""" return self._failure - def fail(self, error: Exception) -> None: + def fail(self, error: Exception, stage: str | None = None) -> None: """Record `error` as this rollout's outcome (captured onto the trace, the remaining stages skipped) — the run's owner reporting a failure the run - itself couldn't see, e.g. its user raising between segments.""" + itself couldn't see, e.g. its user raising between segments. `stage` names + the rollout stage it broke in, when known.""" if ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -237,7 +293,7 @@ def fail(self, error: Exception) -> None: logger.exception("unexpected error in rollout %s", self.trace.id) self._failed = True self._failure = error - self.trace.record_error(error) + self.trace.record_error(error, attempt=self._attempt, stage=stage) def timeout(self, stage: str) -> None: """Record `stage`'s expired deadline as this rollout's outcome: a stop, not @@ -254,7 +310,15 @@ async def open(self) -> bool: self.trace.timing.boot.start = time.time() self.trace.notify() if self._borrowed_runtime is None: - self.runtime = make_backend(self.runtime_config, name=self.trace.id).bind() + config = self.runtime_config + name = self.trace.id + if self._restore is not None: + # A restored trace keeps its id; its new box must not collide with the + # box an earlier attempt or process left behind. + name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" + if self._restore.box: + config = config.model_copy(update={"checkpoint": self._restore.id}) + self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -269,13 +333,24 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info.model_copy() - logger.info( - "rollout start: id=%s task=%s harness=%s runtime=%s", - self.trace.id, - self.task.data.idx, - self.harness.config.name, - self.runtime_config.type, - ) + if self._restore is None: + logger.info( + "rollout start: id=%s task=%s harness=%s runtime=%s", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "rollout restore: id=%s task=%s harness=%s runtime=%s checkpoint=%s turns=%d", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + self._restore.id, + self.trace.num_turns, + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -320,6 +395,15 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() + # A box snapshot needs a runtime that takes them; a harness that keeps + # nothing in its box is restorable from the trace alone (box-less). + self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + if ( + self._checkpointing is not None + and self._borrowed_runtime is None + and (not self._boxless or not self.harness.EXECUTES_CODE) + ): + self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: isolated = self._scoring_policy.mode == "isolated" self._scoring_task = self.task.grading_task( @@ -343,17 +427,21 @@ async def open(self) -> bool: self.trace.timing.boot.end = now self.trace.timing.setup.start = now self.trace.notify() - async with ( - boundary(TaskError, "task setup"), - ): - await invoke( - self.task.setup, - { - "trace": self.trace, - "runtime": runtime, - "deployment": self.deployment, - }, - ) + # A restored box already holds the task's setup; only the harness's + # (idempotent) install runs again. A box-less restore boots a fresh box. + if self._restore is None or not self._restore.box: + async with boundary(TaskError, "task setup"): + await invoke( + self.task.setup, + { + "trace": self.trace, + "runtime": runtime, + "deployment": self.deployment, + }, + ) + if self._restore is not None: + # The trace's state and info are the checkpoint's, not setup's. + self.trace.rewind_state(self._restore) async with ( boundary(HarnessError, "harness setup"), ): @@ -467,9 +555,11 @@ async def open(self) -> bool: harness_data = harness_data.model_copy( update={"system_prompt": network_prompt} ) + # A restored trace already carries its prompt's rewrites. if ( self._session.request_interceptors and harness_data.prompt is not None + and self._restore is None ): prompt = harness_data.prompt system_prompt = harness_data.system_prompt @@ -520,6 +610,7 @@ async def open(self) -> bool: ) else {} ) + self._harness_data = harness_data self._harness_session = await self.harness.session( self.ctx, self.trace, @@ -537,13 +628,14 @@ async def open(self) -> bool: self.timeout("setup") if not self._deployment_started: self.fail( - SandboxError("deployment provisioning exceeded setup timeout") + SandboxError("deployment provisioning exceeded setup timeout"), + stage="setup", ) else: - self.fail(e) + self.fail(e, stage="setup") return False except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure - self.fail(e) + self.fail(e, stage="setup") return False except BaseException: # A cancellation mid-setup kills the driver's await with it, so no @@ -568,15 +660,18 @@ async def step( self, messages: Messages | None = None, *, + intercepted: int = 0, on_input: Callable[[], None] | None = None, ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + The first `intercepted` messages pass to the harness as they are — they were + intercepted when the trace first saw them (a restored checkpoint's tail). + Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -595,25 +690,32 @@ async def step( try: async with deadline: assert self._harness_session is not None - if messages is not None and self._session.request_interceptors: + if ( + messages is not None + and len(messages) > intercepted + and self._session.request_interceptors + ): prepared, rewrites = await self._session.prepare_users( - Request(messages=messages) + Request(messages=messages[intercepted:]) ) - messages = prepared.messages + messages = [*messages[:intercepted], *prepared.messages] self.trace.request_rewrites.extend(rewrites) if self._session.stopped: return False self._session.on_turn_input = on_input assert self.deployment is not None - await self._harness_session.turn(messages) + if await self._turn(messages): + assert self._session.aborted is not None + self.fail(self._session.aborted, stage="agent") + return False except TimeoutError as e: # An expired rollout deadline is the agent breaking its time budget: a - # timeout stop. A TimeoutError from the harness's own I/O with no - # expired deadline stays the raw failure. + # timeout stop. A TimeoutError from the harness's own I/O with no expired + # deadline stays the raw failure. if self.deadline_at is not None and (loop.time() >= self.deadline_at): self.timeout("agent") else: - self.fail(e) + self.fail(e, stage="agent") return False except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure if self._session.stopped: @@ -627,9 +729,9 @@ async def step( real = self._session.error if real is not None and isinstance(e, RolloutError): real.__cause__ = e - self.fail(real) + self.fail(real, stage="agent") else: - self.fail(e) + self.fail(e, stage="agent") return False finally: self._session.on_turn_input = None @@ -649,6 +751,188 @@ async def step( # never moved, forever. return self.ok and trace.num_turns > turns_before + async def _turn(self, messages: Messages | None) -> bool: + """Run one harness segment. A request hook that fails ends it at once (the + session aborts): the harness's turn is cancelled, so the harness never handles + the failure. Returns whether the segment was aborted.""" + assert self._harness_session is not None + turn = asyncio.ensure_future(self._harness_session.turn(messages)) + aborted = asyncio.ensure_future(self._session.abort_event.wait()) + try: + await asyncio.wait({turn, aborted}, return_when=asyncio.FIRST_COMPLETED) + finally: + aborted.cancel() + if not turn.done(): + turn.cancel() + await asyncio.wait({turn}) + if self._session.aborted is not None: + if not turn.cancelled(): + turn.exception() # the abort decides the outcome, whatever the turn did + return True + turn.result() + return False + + async def restore(self) -> bool: + """Run the restored attempt's first segment: a new harness process in the box + restored from the checkpoint continues the conversation the checkpoint left + off at (`HarnessSession.restore`), told that the box was restored.""" + checkpoint = self._restore + assert checkpoint is not None and self._harness_data is not None + assert self._harness_session is not None + # The relaunch re-emits the session's system prompt (the task's, plus a + # network notice) through the program's own system message; the tail's copy + # would double it. + system = self._harness_data.system_prompt + tail = [m for m in checkpoint.tail if m.role != "system" or system is None] + self._session.pinned_tool_results.update(checkpoint.pins) + self._session.mark_intercepted(self.trace.messages, tail) + messages = self._harness_session.restore(checkpoint.harness, tail) + # Only the restore notice is new; user hooks prepare it like any user turn. + return await self.step(messages, intercepted=len(messages) - 1) + + async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: + if self._closed: + # Settling may have drained `_captures` already: a snapshot taken now + # would start its upload after the box is gone. + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + return await self._capture(turn) + finally: + self._captures.discard(task) + + async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: + assert self.runtime is not None + assert self._checkpointing is not None + if self.trace.num_turns % self._checkpointing.interval: + return None + if len(self.trace._pending) > 1: + # Another request is in flight (parallel sub-agents share the trace): the + # box is not idle, so no snapshot would match this turn's prefix. + return + assert self._harness_session is not None + try: + harness = await self._harness_session.checkpoint_state() + except Exception: + logger.warning( + "harness checkpoint state failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + return + if harness is None: + return + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(self._session.pinned_tool_results) + started = time.time() + if self._boxless: + checkpoint_id = uuid.uuid4().hex + else: + try: + checkpoint_id = await self.runtime.backend.checkpoint() + except Exception: + logger.warning( + "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + ) + return None + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + checkpoint = Checkpoint( + id=checkpoint_id, + node=node, + tail=list(turn.tail), + state=state, + info=info, + pins=pins, + harness=harness, + box=not self._boxless, + time=TimeSpan(start=started), + captured=time.time(), + ) + self.trace.checkpoints.append(checkpoint) + if self._boxless: + # Nothing to upload: restorable as soon as it is recorded. + checkpoint.time.end = checkpoint.captured + self.trace.notify() + else: + self._durable[checkpoint_id] = asyncio.create_task( + self._make_durable(checkpoint) + ) + logger.debug( + "checkpoint %s taken: id=%s node=%s", checkpoint_id, self.trace.id, node + ) + return checkpoint + + async def _make_durable(self, checkpoint: Checkpoint) -> None: + assert self.runtime is not None + try: + checkpoint.bytes = await self.runtime.backend.wait_checkpoint(checkpoint.id) + except Exception: + # Not restorable: no retry may pick it. The backend still holds it as + # stray and frees it with the box. + logger.warning( + "checkpoint %s never became durable (rollout %s)", + checkpoint.id, + self.trace.id, + exc_info=True, + ) + with contextlib.suppress(ValueError): + self.trace.checkpoints.remove(checkpoint) + else: + checkpoint.time.end = time.time() + self.trace.notify() + await self._prune() + finally: + self._durable.pop(checkpoint.id, None) + + async def _prune(self) -> None: + """Delete the snapshots the `keep` policy drops now that a newer one is + restorable, and the ones a restore rewound past.""" + assert self.runtime is not None and self._checkpointing is not None + async with self._pruning: + keep = kept(self.trace.checkpoints, self._checkpointing.keep) + drop = [ + *( + c.id + for c in self.trace.checkpoints + if c.box and c.time.end and not c.deleted and c.id not in keep + ), + *( + c + for c in self.rewound_checkpoints + if c not in self.deleted_checkpoints + ), + ] + if drop: + self.record_deleted( + await self.runtime.backend.discard_checkpoints(drop) + ) + + def record_deleted(self, ids: list[str]) -> None: + """Mark the snapshots with these ids deleted.""" + self.deleted_checkpoints.update(ids) + for checkpoint in self.trace.checkpoints: + if checkpoint.id in self.deleted_checkpoints: + checkpoint.deleted = True + + async def _settle_checkpoints(self, *, wait: bool) -> None: + """Before the box goes: wait for the snapshots still uploading, so a retry + can restore them (its teardown would fail them), or, with nothing to retry, + stop waiting and leave them to be freed.""" + # A snapshot still being taken would start its upload after this returned. + await asyncio.gather(*self._captures, return_exceptions=True) + tasks = list(self._durable.values()) + if not wait: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + # A task cancelled before it first ran never reached its own cleanup. + for checkpoint_id, task in list(self._durable.items()): + if task.done(): + del self._durable[checkpoint_id] + async def _cleanup_harness(self, runtime: Runtime) -> None: if self._harness_cleaned: return @@ -666,6 +950,7 @@ async def abort(self) -> None: (a cancellation mid-setup, a lifetime bug raised to the caller) means the driver will never reach `close()`. Safe after a partial `close()`.""" self._closed = True + await self._settle_checkpoints(wait=False) if self._harness_session is not None: with contextlib.suppress(Exception): await self._harness_session.close() @@ -684,6 +969,24 @@ async def abort(self) -> None: ): with contextlib.suppress(Exception): await self.runtime.backend.stop() + if self.runtime is not None: + # Snapshots kept `after` the run belong to their owner, an aborted run's too; + # strays were never restorable, so nobody else knows of them. + owned = self._checkpointing is None or self._checkpointing.after == "delete" + with contextlib.suppress(Exception): + self.record_deleted( + await self.runtime.backend.discard_checkpoints( + [ + *(self.rewound_checkpoints if owned else ()), + *( + c.id + for c in self.trace.checkpoints + if owned and c.box and not c.deleted + ), + *self.runtime.backend.stray_checkpoints, + ] + ) + ) async def _score_harness(self, runtime: Runtime, deadline: float | None) -> None: """Collect best-effort metrics without replacing a task failure.""" @@ -942,7 +1245,7 @@ async def close(self) -> Trace: trace.timing.scoring.end = time.time() trace.notify() except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure - self.fail(e) + self.fail(e, stage="scoring" if trace.timing.scoring.start else "finalize") finally: if self._harness_session is not None: with contextlib.suppress(Exception): @@ -964,6 +1267,9 @@ async def close(self) -> Trace: trace.split_agent_time() if runtime is not None: await self._cleanup_harness(runtime) + # A failed attempt's retry restores the newest snapshot, which must be + # durable before the box goes. + await self._settle_checkpoints(wait=not trace.ok) # Tear down here — the env's `score()` (later) needs only the traces, # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. @@ -977,6 +1283,10 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() + if runtime is not None and runtime.backend.stray_checkpoints: + await runtime.backend.discard_checkpoints( + runtime.backend.stray_checkpoints + ) except Exception: logger.warning("runtime teardown failed", exc_info=True) logger.info( @@ -985,7 +1295,9 @@ async def close(self) -> Trace: self.task.data.idx, trace.reward, trace.num_turns, - trace.last_error.type if trace.last_error else trace.stop_condition, + trace.last_error.type + if trace.last_error and not trace.ok + else trace.stop_condition, f" dropped={trace.dropped!r}" if trace.is_dropped else "", ) return trace diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 19a004c1cb..a2e8df945c 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -9,6 +9,7 @@ import math import shlex import tempfile +import time import uuid from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager @@ -51,6 +52,9 @@ _OUTPUT_RETRIES = 10 """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it +after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" """Domain of the public URLs `https://-.` that a LAN network's `internet` rule publishes ports at.""" @@ -150,6 +154,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -242,8 +247,8 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - **source, environment_vars=self.env, + **source, **{k: v for k, v in options.items() if v is not None}, ) ) @@ -266,7 +271,12 @@ async def create_and_capture_id(): ) await self._client.wait_for_creation(self.info.id, max_attempts=900) logger.info( - "prime: sandbox %s up (image=%s)", self.info.id, self.config.image + "prime: sandbox %s up (image=%s%s)", + self.info.id, + self.config.image, + f", checkpoint={self.config.checkpoint}" + if self.config.checkpoint + else "", ) await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" @@ -283,6 +293,56 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def checkpoint(self) -> str: + # The platform fixes the snapshot's content before the request returns, then + # uploads it. Until it is durable the snapshot is stray: the platform refuses + # to delete it while this box lives, so teardown frees it after. + async def create() -> str: + checkpoint = await self._client.checkpoint(self.info.id) + self.stray_checkpoints.append(checkpoint.id) + return checkpoint.id + + try: + return await run_shielded(create()) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + started = time.monotonic() + try: + durable = await self._client.wait_for_checkpoint( + checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) + except Exception as e: + raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e + self.stray_checkpoints.remove(checkpoint) + logger.debug( + "prime: checkpoint %s durable after %.1fs on sandbox %s", + checkpoint, + time.monotonic() - started, + self.info.id, + ) + return durable.stored_bytes + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + # The platform refuses only the newest checkpoint of a running box (the parent + # of its live state); older ones go while it runs, their data kept as long as a + # newer one needs it. + from prime_sandboxes import AsyncSandboxClient + + deleted = [] + async with AsyncSandboxClient() as client: + for checkpoint in ids: + try: + await client.delete_checkpoint(checkpoint) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + logger.warning( + "prime: failed to delete checkpoint %s: %s", checkpoint, e + ) + else: + deleted.append(checkpoint) + return deleted + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index b0c1d32756..7a917b6f81 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -184,6 +184,12 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + """Whether `checkpoint` snapshots the box's filesystem for a new box of this config + to boot from (its config's `checkpoint`). A rollout checkpoints before each model + turn, so a retry restores the newest turn instead of replaying the run (see + `AgentConfig.checkpoint`).""" + info: BaseRuntimeInfo def __init__(self, name: str | None = None) -> None: @@ -192,6 +198,9 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() + self.stray_checkpoints: list[str] = [] + """Checkpoints this box took that never became restorable (a failed or cancelled + durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] self.stopped = False @@ -218,6 +227,24 @@ def type(self) -> str: async def start(self) -> None: pass + async def checkpoint(self) -> str: + """Snapshot the filesystem and return the checkpoint's id once its content is + fixed: writes after this returns are not in it. It becomes restorable only later + (`wait_checkpoint`); the box keeps running.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + """Wait until `checkpoint` is restorable and return its stored size in bytes, + when the provider reports it. Raises if it never becomes restorable. The box + must stay alive until then: its teardown fails a pending checkpoint.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + """Free checkpoints nothing will restore from and return the ids it freed. + Best-effort: a provider may refuse one a live box still needs (prime: the + newest one of a running box).""" + return [] + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index a0ec7f120f..2b9bb4043d 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -34,7 +34,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import HarnessError, RolloutError, TaskError from verifiers.v1.harnesses.utils.compaction import bound_tool_message -from verifiers.v1.trace import InterceptRecord, Trace +from verifiers.v1.trace import Checkpoint, InterceptRecord, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -184,6 +184,10 @@ class RolloutSession: when its program dies on it the rollout records this original error instead of a secondary `HarnessError`. A harness that completes cleanly after the failure handled it. Reset before each model turn, so a successful retry clears it.""" + aborted: "RolloutError | None" = None + """A request hook's failure: the task, not the provider, refused a turn. The rollout + ends the running segment with it (`Rollout.step`); the harness never sees it.""" + abort_event: asyncio.Event = field(default_factory=asyncio.Event) idempotent_requests: dict[str, IdempotentRequest] = field(default_factory=dict) """Explicit keys or marked SDK retries mapped to their replay state.""" released: bool = False @@ -209,6 +213,12 @@ class RolloutSession: gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" prepared_users: Counter[str] = field(default_factory=Counter) + checkpointer: Callable[[graph.PendingTurn], Awaitable[Checkpoint | None]] | None = ( + None + ) + """Snapshots the rollout's box as a turn goes upstream, alongside the model call; + the reply waits for the capture (`Rollout` sets it when its runtime supports + checkpoints and a retry may restore one). Returns the checkpoint it took, if any.""" @property def stopped(self) -> bool: @@ -385,6 +395,21 @@ async def rewrite_request( self.pinned_tool_results[tool_keys[position]] = after return current, records, None + def mark_intercepted(self, branch: Messages, tail: Messages) -> None: + """Treat `tail`, which follows `branch` and was intercepted in an earlier + attempt (a restored checkpoint's), as prepared: the harness re-sends it, and + a non-idempotent hook must not rewrite it twice.""" + request = Request(messages=[*branch, *tail]) + for position, key in self.tool_result_keys(request).items(): + message = request.messages[position] + if position >= len(branch) and isinstance(message, ToolMessage): + self.pinned_tool_results.setdefault(key, message) + self.prepared_users.update( + graph.message_hash(message) + for message in tail + if isinstance(message, UserMessage) + ) + def consume_prepared(self, messages: Messages) -> None: """Forget pre-harness user rewrites only after their model request commits.""" for message in messages: @@ -605,6 +630,12 @@ def adopt(self, task: "asyncio.Task | None") -> None: self.tasks.add(task) task.add_done_callback(self.tasks.discard) + def abort(self, error: "RolloutError") -> None: + """End the running segment with `error`. The first abort wins.""" + if self.aborted is None: + self.aborted = error + self.abort_event.set() + def release(self) -> None: """Seal the session: no further trace mutation, and in-flight handlers cancel.""" self.released = True diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index f526c17d5a..b2de785353 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -250,6 +250,8 @@ class TimeSplit(BaseModel): class AgentSpan(TimeSpan): model: TimeSplit = Field(default_factory=TimeSplit) harness: TimeSplit = Field(default_factory=TimeSplit) + checkpoint: TimeSplit = Field(default_factory=TimeSplit) + """Time the model replies waited for checkpoint captures (`Checkpoint.held`).""" class Timing(BaseModel): @@ -266,6 +268,10 @@ class Error(BaseModel): message: str status_code: int | None = None traceback: str | None = None + attempt: int = 0 + """The run attempt that captured it: 0 for the first, then one per retry.""" + stage: str | None = None + """The rollout stage that failed (`setup`, `agent`, `finalize`, `scoring`), when known.""" class VersionInfo(BaseModel): @@ -369,6 +375,104 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's box taken as a model turn went upstream: the box holds + the effects of every message in `node`'s conversation plus `tail`, and nothing of the + turn that followed. A retry restores it (`Trace.rewind`).""" + + id: str + """The runtime's checkpoint id (`Runtime.checkpoint`), or this record's own for a + box-less checkpoint.""" + box: bool = True + """Whether a runtime snapshot holds the box. A box-less checkpoint (a harness that + keeps nothing in its box, on a runtime without snapshots) restores into a fresh box + whose task setup runs again; the trace and its typed state carry the rest.""" + node: int | None = None + """The last committed node whose conversation the box's state follows; None when + the snapshot preceded the first turn.""" + tail: Messages = Field(default_factory=list, exclude=True) + """The turn's prompt messages after `node` that had no node yet (tool results, user + turns): with `node`'s conversation, the prompt a restored harness continues from. + Live-only — a retry restores it in the process that took the snapshot.""" + state: Any = Field(None, exclude=True) + """The trace's typed state as the snapshot was taken (tool servers write it over + `/state`), restored with the box so a retry sees one consistent world. Live-only.""" + info: dict[str, Any] | None = Field(None, exclude=True) + """`Trace.info` as the snapshot was taken, restored with `state`. Live-only.""" + pins: dict[tuple[str, str], ToolMessage] = Field(default_factory=dict, exclude=True) + """The rollout's pinned tool results as the snapshot was taken + (`RolloutSession.pinned_tool_results`): a restored harness re-sends its own + copies, which must be pinned again to match the trace. Live-only.""" + time: TimeSpan = Field(default_factory=TimeSpan) + """From the snapshot request until it was durable (restorable); `end` stays 0 while + it is not.""" + captured: float = 0.0 + """When the request returned: the snapshot's content was fixed from then on.""" + held: float = 0.0 + """Seconds the turn's reply waited for the capture; 0 when the model call took + longer, since the snapshot runs alongside it.""" + bytes: int | None = None + """The snapshot's stored size, which the provider bills, once it is durable.""" + harness: dict[str, Any] = Field(default_factory=dict) + """What the harness session needs to continue in a box restored from the snapshot + (`HarnessSession.checkpoint_state`), e.g. an ACP agent's session id.""" + restored: bool = False + """Whether a retry restored the run from this snapshot.""" + deleted: bool = False + """Whether the snapshot was deleted (`RolloutCheckpointConfig.keep`, `.after`): it + can no longer be restored.""" + + +class PinnedResult(BaseModel): + """One pinned tool result (`RolloutSession.pinned_tool_results`), keyed by the + conversation prefix that issued its call and the call's id.""" + + prefix: str + call_id: str + message: ToolMessage + + +class RestorePoint(BaseModel): + """A checkpoint with everything a restore needs, serializable: its live-only fields + travel with it, so another process (an orchestrator resuming its run) can restore + the trace from it (`RunRequest.restore`).""" + + checkpoint: Checkpoint + tail: Messages = Field(default_factory=list) + state: dict[str, Any] | None = None + info: dict[str, Any] | None = None + pins: list[PinnedResult] = Field(default_factory=list) + + @classmethod + def of(cls, checkpoint: Checkpoint) -> RestorePoint: + state = checkpoint.state + return cls( + checkpoint=checkpoint, + tail=list(checkpoint.tail), + state=state.model_dump(mode="json") + if isinstance(state, BaseModel) + else state, + info=checkpoint.info, + pins=[ + PinnedResult(prefix=prefix, call_id=call_id, message=message) + for (prefix, call_id), message in checkpoint.pins.items() + ], + ) + + def live(self, state_type: type[BaseModel]) -> Checkpoint: + """The checkpoint with its live-only fields back, the state as `state_type`.""" + return self.checkpoint.model_copy( + update={ + "tail": list(self.tail), + "state": state_type.model_validate(self.state) + if self.state is not None + else None, + "info": self.info, + "pins": {(p.prefix, p.call_id): p.message for p in self.pins}, + } + ) + + def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -603,6 +707,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry restores the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -643,6 +750,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -883,6 +998,19 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None + @property + def restorable(self) -> Checkpoint | None: + """The newest checkpoint a restore can use: restorable and not deleted.""" + return next( + (c for c in reversed(self.checkpoints) if c.time.end and not c.deleted), + None, + ) + + @property + def has_restored(self) -> bool: + """Whether a retry restored this run from a checkpoint rather than starting over.""" + return any(checkpoint.restored for checkpoint in self.checkpoints) + @property def transcript(self) -> str: """Text transcript of the last branch's messages; tool outputs are omitted.""" @@ -957,13 +1085,22 @@ def drop(self, reason: str) -> None: self.dropped = reason def split_agent_time(self) -> None: - """Split the agent span into model and harness time.""" + """Split the agent span into model, checkpoint and harness time.""" span = self.timing.agent if not span.end: return - model = sum(call.time.duration for call in self.calls) - span.model.duration = min(model, span.duration) - span.harness.duration = span.duration - span.model.duration + # A restored trace keeps the calls and checkpoints of the attempts before + # this span. + calls = sum( + call.time.duration for call in self.calls if call.time.start >= span.start + ) + # A call's span includes the time its reply waited for the capture. + held = sum(c.held for c in self.checkpoints if c.captured >= span.start) + span.checkpoint.duration = min(held, calls, span.duration) + span.model.duration = min(calls, span.duration) - span.checkpoint.duration + span.harness.duration = ( + span.duration - span.model.duration - span.checkpoint.duration + ) def record_timeout(self, stage: str) -> None: """Record a stage deadline's expiry, and stop the trace as `_timeout`. @@ -972,7 +1109,52 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) - def record_error(self, error: Exception) -> None: + def rewind_state(self, checkpoint: Checkpoint) -> None: + """Set the typed state and `info` back to `checkpoint`'s.""" + if checkpoint.state is not None: + self.state = copy.deepcopy(checkpoint.state) + if checkpoint.info is not None: + self.info = copy.deepcopy(checkpoint.info) + + def rewind(self, checkpoint: Checkpoint) -> list[str]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and every call stay as the attempts' history (the + calls of the dropped nodes unlinked, so usage still counts them). Returns the ids + of the later box snapshots this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + del self.nodes[keep:] + for node in self.nodes: + node.semantic_parents = [ + link for link in node.semantic_parents if link.node < keep + ] + self._head_index = {} + self._pending = {} + self._generation += 1 + for call in self.calls: + if call.node is not None and call.node >= keep: + call.node = None + self.rewind_state(checkpoint) + # The restored rollout re-stamps every stage span; the run's start stays. + self.timing = Timing(start=self.timing.start) + rewound = [c for c in self.checkpoints if c.node is not None and c.node >= keep] + self.checkpoints = [c for c in self.checkpoints if c not in rewound] + dropped = [c.id for c in rewound if c.box] + self.rewards = {} + self.metrics = {} + self.dropped = None + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + + def record_error( + self, error: Exception, *, attempt: int = 0, stage: str | None = None + ) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" self.errors.append( @@ -980,6 +1162,8 @@ def record_error(self, error: Exception) -> None: type=type(error).__name__, message=str(error), status_code=getattr(error, "status_code", None), + attempt=attempt, + stage=stage, # Provider errors already carry the actionable upstream diagnostic. # Keep full tracebacks for every other failure. traceback=None From 824ca302b1db9c038125ba0ea409156665808031 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 22:40:33 +0000 Subject: [PATCH 12/53] narrow checkpoints to retries and name snapshots Scope rollout checkpointing to retries within one Agent.run: drop the cross-process restore (Env and Agent.run restore params, RestorePoint, PinnedResult, Trace.restorable, Agents.only). Call the runtime side a snapshot: backends return a typed Snapshot (id, type) from snapshot(), wait_snapshot() and discard_snapshots() take it, and restore_config() builds the config a new box boots from. Runtimes and harnesses declare SUPPORTS_FILESYSTEM_SNAPSHOTS, and RolloutCheckpointConfig.type takes a SnapshotType, so memory snapshots can slot in behind the same API. Also from review: - snapshot every 10th turn by default - copy state before the first await of a capture, so response hooks that run meanwhile do not leak into the checkpoint - trim the checkpoints after the restored one by position, not node - build the rollout params once per run, so an RLM retry keeps its skill home and the restored session loads - discard snapshots on the shared leased client Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 2 +- docs/harnesses.md | 2 +- tests/unit/test_checkpoint.py | 57 ++++++------ tests/unit/test_retry_ids.py | 4 +- verifiers/v1/__init__.py | 6 ++ verifiers/v1/agent.py | 80 ++++++----------- verifiers/v1/configs/agent.py | 44 ++++----- verifiers/v1/configs/runtime.py | 4 + verifiers/v1/env.py | 21 +---- verifiers/v1/harness.py | 4 +- verifiers/v1/harnesses/bash/harness.py | 2 +- verifiers/v1/harnesses/claude_code/harness.py | 2 +- verifiers/v1/harnesses/codex/harness.py | 2 +- verifiers/v1/harnesses/null/harness.py | 2 +- verifiers/v1/harnesses/rlm/harness.py | 6 +- verifiers/v1/rollout.py | 82 +++++++++-------- verifiers/v1/runtimes/__init__.py | 4 + verifiers/v1/runtimes/backends/prime.py | 42 +++++---- verifiers/v1/runtimes/base.py | 55 +++++++----- verifiers/v1/trace.py | 90 ++++--------------- 20 files changed, 222 insertions(+), 289 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 7ef3334272..4b775c23c1 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,4 +77,4 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box as model turns go upstream, and a retry restores the newest snapshot and continues the trace from there. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_CHECKPOINTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `interval` sets how often it snapshots, `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box at model turns, and a retry restores the newest snapshot and continues the trace from there. Only retries within one `Agent.run` restore; a new process starts over. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_SNAPSHOTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `type` sets what a snapshot captures (`"filesystem"`, the only type today), `interval` how often it snapshots (every 10th turn by default), `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. diff --git a/docs/harnesses.md b/docs/harnesses.md index e634a3e746..2144e499ff 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_CHECKPOINTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem checkpoint, so a retry restores the newest checkpoint instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. +- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem snapshot, so a retry restores the newest snapshot instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 19b1918330..414c877189 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -16,7 +16,6 @@ from verifiers.v1.trace import ( Checkpoint, ModelCall, - RestorePoint, TimeSpan, TraceTask, ) @@ -67,6 +66,16 @@ def test_rewind_keeps_the_dropped_turns_calls(): assert [call.node for call in trace.calls] == [None] +def test_rewind_drops_the_later_checkpoints_at_the_same_node(): + trace = make_trace() + older = Checkpoint(id="older", node=1, snapshot=vf.Snapshot(id="older")) + target = Checkpoint(id="target", node=1, snapshot=vf.Snapshot(id="target")) + later = Checkpoint(id="later", node=1, snapshot=vf.Snapshot(id="later")) + trace.checkpoints.extend([older, target, later]) + assert trace.rewind(target) == [vf.Snapshot(id="later")] + assert [c.id for c in trace.checkpoints] == ["older", "target"] + + def test_restored_span_counts_only_its_own_calls(): trace = make_trace() trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 @@ -128,17 +137,18 @@ def __init__(self) -> None: self.taken = 0 self.deleted: list[str] = [] - async def checkpoint(self) -> str: + async def snapshot(self) -> vf.Snapshot: self.taken += 1 - return f"ck{self.taken}" + return vf.Snapshot(id=f"ck{self.taken}") - async def wait_checkpoint(self, checkpoint: str) -> int | None: + async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: await self.durable.wait() if self.fail: raise RuntimeError("upload failed") return 1024 - async def discard_checkpoints(self, ids: list[str]) -> list[str]: + async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: + ids = [snapshot.id for snapshot in snapshots] self.deleted += ids return ids @@ -161,8 +171,8 @@ async def checkpoint_state() -> dict: run._captures = set() run._closed = False run._pruning = asyncio.Lock() - run.deleted_checkpoints = set() - run.rewound_checkpoints = [] + run.deleted_snapshots = set() + run.rewound_snapshots = [] run._boxless = False return run, backend @@ -220,7 +230,12 @@ async def test_a_closing_run_takes_no_checkpoint(): def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: return [ - Checkpoint(id=i, time=TimeSpan(start=1.0, end=2.0), deleted=i in deleted) + Checkpoint( + id=i, + snapshot=vf.Snapshot(id=i), + time=TimeSpan(start=1.0, end=2.0), + deleted=i in deleted, + ) for i in ids ] @@ -239,40 +254,22 @@ def test_keep_policies(): async def test_a_durable_snapshot_prunes_the_older_ones(): run, backend = make_rollout(keep="last") run.trace.checkpoints = durable("old1", "old2") - run.rewound_checkpoints = ["rewound"] + run.rewound_snapshots = [vf.Snapshot(id="rewound")] backend.durable.set() newest = await run._checkpoint(TURN) await run._settle_checkpoints(wait=True) assert sorted(backend.deleted) == ["old1", "old2", "rewound"] assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] - assert run.deleted_checkpoints == {"old1", "old2", "rewound"} + assert run.deleted_snapshots == {"old1", "old2", "rewound"} async def test_a_boxless_checkpoint_is_restorable_at_once(): run, backend = make_rollout() run._boxless = True checkpoint = await run._checkpoint(TURN) - assert checkpoint is not None and not checkpoint.box + assert checkpoint is not None and checkpoint.snapshot is None assert checkpoint.time.end and backend.taken == 0 - assert run.trace.restorable is checkpoint - - -def test_a_restore_point_round_trips(): - checkpoint = Checkpoint( - id="ck", - node=1, - tail=[ToolMessage(tool_call_id="c1", content="out")], - state=vf.State(), - info={"nonce": "n"}, - pins={("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned")}, - time=TimeSpan(start=1.0, end=2.0), - ) - wire = RestorePoint.of(checkpoint).model_dump(mode="json") - back = RestorePoint.model_validate(wire).live(vf.State) - assert back.tail == checkpoint.tail - assert back.info == {"nonce": "n"} - assert back.pins == checkpoint.pins - assert back.state == checkpoint.state + assert run.trace.checkpoints[-1] is checkpoint def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 7c3d6f9613..45c540ecf8 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -36,8 +36,8 @@ class FakeRollout: runtime = None def __init__(self, *, on_trace, trace_id, attempt, **_): - self.rewound_checkpoints: list[str] = [] - self.deleted_checkpoints: set[str] = set() + self.rewound_snapshots: list[str] = [] + self.deleted_snapshots: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 33ff9dd52b..f71349fef3 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -95,6 +95,8 @@ Runtime, RuntimeInfo, RuntimeProcess, + Snapshot, + SnapshotType, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -279,6 +281,8 @@ "RuntimeConfig": "verifiers.v1.configs.runtime", "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", + "Snapshot": "verifiers.v1.runtimes", + "SnapshotType": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplingMask": "verifiers.v1.types", @@ -489,6 +493,8 @@ "RuntimeProcess", "RuntimeConfig", "RuntimeInfo", + "Snapshot", + "SnapshotType", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 449ce0718d..5fdab5eacc 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Generic, Literal, Self, cast +from typing import Any, Generic, Literal, Self, cast from weakref import WeakValueDictionary from typing_extensions import TypeVar @@ -39,6 +39,7 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, backend_cls, provision_backend, runtime_is_local, @@ -575,9 +576,6 @@ def __init__( sampling=config.sampling, ) self._closed = False - self.pending_restore: tuple[Trace, Checkpoint] | None = None - """A restore the next `run` starts from (a trace another process ran); consumed - by that run.""" self.runtime_config: RuntimeConfig = config.runtime self.interception = interception self.limits = RolloutLimits( @@ -656,8 +654,8 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if harness.SUPPORTS_FILESYSTEM_CHECKPOINTS and ( - backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + if harness.SUPPORTS_FILESYSTEM_SNAPSHOTS and ( + backend.SUPPORTS_FILESYSTEM_SNAPSHOTS or (config.boxless and not harness.EXECUTES_CODE) ): return config @@ -666,7 +664,7 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: _checkpointing_warned.add(key) reason = ( f"harness {harness.config.id!r} does not support it" - if not harness.SUPPORTS_FILESYSTEM_CHECKPOINTS + if not harness.SUPPORTS_FILESYSTEM_SNAPSHOTS else f"the {self.config.runtime.type!r} runtime takes no snapshots" ) logger.warning( @@ -702,7 +700,6 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -721,27 +718,23 @@ async def run( the selected verifier workspace, inside its timeout and retry boundary.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") - if restore is None and self.pending_restore is not None: - restore, self.pending_restore = self.pending_restore, None retry = self.config.retries retry_state = RetryState(retry) checkpointing = self._checkpointing() if runtime is None else None - # One id for the run across its attempts; a restored trace keeps its own. - trace_id = restore[0].id if restore is not None else uuid.uuid4().hex + trace_id = uuid.uuid4().hex # one id for the run across its attempts history: list[Error] = [] # errors of the attempts started over from - # `restore` continues a trace another process ran (an orchestrator resuming its - # run): its first attempt restores it, and `on_trace` sees it as the run's trace. - trace: Trace | None = restore[0] if restore is not None else None - target: Checkpoint | None = restore[1] if restore is not None else None - if restore is not None and on_trace is not None: - on_trace(restore[0]) - snapshots: set[str] = set() # every checkpoint any attempt took + trace: Trace | None = None + target: Checkpoint | None = None # the checkpoint the next attempt restores + snapshots: dict[str, Snapshot] = {} # every box snapshot any attempt took deleted: set[str] = set() failed_restores: set[str] = set() run: Rollout | None = None # The `finally` also covers a cancellation between attempts, when no rollout is # open to delete them on abort. budget = retry_state.max_retries + # Built once: every attempt runs the same harness instance, so per-run harness + # state (RLM's skill home) is the one a restored box holds. + params = self._rollout_params(task, runtime, dict(tools or {})) attempt = 0 try: while True: @@ -756,7 +749,7 @@ async def run( run = await self._run_once( task, runtime, - tools, + params, on_trace, collect_artifacts, on_score, @@ -766,9 +759,11 @@ async def run( checkpoint=checkpointing, ) trace = run.trace - snapshots.update(run.rewound_checkpoints) - snapshots.update(c.id for c in trace.checkpoints if c.box) - deleted.update(run.deleted_checkpoints) + snapshots.update((s.id, s) for s in run.rewound_snapshots) + snapshots.update( + (c.snapshot.id, c.snapshot) for c in trace.checkpoints if c.snapshot + ) + deleted.update(run.deleted_snapshots) if target is not None and any( error.stage == "setup" for error in trace.errors[seen:] ): @@ -776,25 +771,6 @@ async def run( failed_restores.add(target.id) if trace.ok: break - if ( - attempt == 0 - and restore is not None - and target is not None - and target.id in failed_restores - ): - # Another process's restore that cannot boot (its snapshot is gone) - # is not the rollout's failure: start over, unbudgeted. - logger.warning( - "restoring trace %s from checkpoint %s failed; starting over", - trace.id, - target.id, - ) - history.extend(trace.errors) - target = None - trace = None - attempt += 1 - budget += 1 - continue if attempt >= budget: break if runtime is not None: @@ -839,13 +815,15 @@ async def run( and checkpointing.after == "keep" else set() ) - drop = sorted(snapshots - deleted - keep) + drop = [ + snapshot + for snapshot_id, snapshot in sorted(snapshots.items()) + if snapshot_id not in deleted and snapshot_id not in keep + ] if drop and run is not None and run.runtime is not None: try: run.record_deleted( - await run_shielded( - run.runtime.backend.discard_checkpoints(drop) - ) + await run_shielded(run.runtime.backend.discard_snapshots(drop)) ) except Exception: logger.warning( @@ -864,7 +842,7 @@ async def _run_once( self, task: Task, runtime: Runtime | None, - shared_tools: Mapping[str, SharedToolServer] | None, + params: dict[str, Any], on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None, @@ -874,7 +852,6 @@ async def _run_once( restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, ) -> Rollout: - params = self._rollout_params(task, runtime, dict(shared_tools or {})) if ( collect_artifacts and not backend_cls(params["runtime_config"]).SUPPORTS_TASK_IMAGES @@ -1138,7 +1115,6 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: async with self._gate or nullcontext(): trace = await super().run( @@ -1148,7 +1124,6 @@ async def run( on_trace=self._watch(on_trace), collect_artifacts=collect_artifacts, on_score=self._scoring_callback(on_score), - restore=restore, ) self._completed.append(trace) return trace @@ -1216,11 +1191,6 @@ def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None: for name, value in agent_config_fields(config).items() } - def only(self) -> AgentT | None: - """The agent of a config with one agent seat; None when it has several.""" - agents = list(self._agents.values()) - return agents[0] if len(agents) == 1 else None - def __getattr__(self, name: str) -> AgentT: # self.__dict__ directly: attribute lookup re-entering __getattr__ before # __init__ ran (copy/unpickle) must raise, not recurse. diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index c9aa905980..833262f827 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -17,7 +17,7 @@ from verifiers.v1.configs.deployment import DeploymentConfig from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig +from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig, SnapshotType from verifiers.v1.configs.sampling import SamplingConfig from verifiers.v1.configs.scoring import ScoringConfig from verifiers.v1.utils.generic import merge_defaults @@ -40,36 +40,26 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing. Snapshot the rollout's box (its filesystem) as model - turns go upstream, so a retry restores the newest snapshot instead of starting - over: the trace keeps its turns up to there, and a new harness process continues - in a box restored to that state, told that the box was restored. Needs a runtime - with checkpoints (prime) and a harness that can restore one (both set - `SUPPORTS_FILESYSTEM_CHECKPOINTS`); elsewhere the agent warns once and a retry starts over. - The snapshot runs alongside the model call, and the reply waits only until its - content is fixed. - - Known limits: a harness with concurrent sub-agents can be snapshotted while one of - them is mid-tool, and processes the agent left running (background jobs, - sub-agents, a REPL's variables) are gone after a restore, which cold boots.""" - - interval: PositiveInt = 1 - """Snapshot before every this-many-th model turn (the 1st, the 1+n-th, ...). A - larger interval takes fewer snapshots, and a restore loses up to this many turns.""" + """Rollout checkpointing: snapshot the box at model turns, so a retry continues + from the newest snapshot instead of starting over. Needs a runtime and a harness + that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts over. Processes + the agent left running (background jobs, a REPL) are gone after a restore.""" + + type: SnapshotType = "filesystem" + """What a snapshot captures.""" + interval: PositiveInt = 10 + """Snapshot at every n-th model turn (the 1st, the 1+n-th, ...); a restore loses + up to n turns.""" keep: Literal["all", "last"] | PositiveInt = "last" - """Which snapshots stay while the rollout runs: `"last"` keeps only the newest - restorable one, `"all"` keeps every one, and `k` keeps every k-th (the 1st, the - 1+k-th, ...) plus the newest. The others are deleted as soon as a newer one is - restorable. A retry whose restore fails falls back to an older snapshot that is - still kept, else it starts over.""" + """Snapshots kept while the rollout runs: the newest (`"last"`), every one + (`"all"`), or every k-th plus the newest (`k`). A failed restore falls back to an + older kept one.""" after: Literal["delete", "keep"] = "delete" - """What happens to the kept snapshots when the rollout's run is final (it succeeded, - or its retries ran out): `"delete"` deletes them once the box is gone; `"keep"` - leaves them for their owner (forks, inspection), who must delete them.""" + """What happens to the kept snapshots once the run is final: `"delete"` them, or + `"keep"` them for their owner to delete.""" boxless: bool = False """Also checkpoint a harness that runs no code (null) on a runtime without - snapshots, from the trace alone. Off by default: such a run records a checkpoint - on its trace at every snapshot turn.""" + snapshots, from the trace alone.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 2ac1fcdc4e..716dadb1ce 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -14,6 +14,10 @@ parse_network_rule, ) +SnapshotType = Literal["filesystem"] +"""What a runtime snapshot captures: `filesystem` holds the box's files, not its +processes.""" + class BindMount(BaseConfig): """An existing file or directory on the container engine's host.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 2223174e5c..1f943574fa 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -32,7 +32,7 @@ from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task -from verifiers.v1.trace import Checkpoint, Error, Trace, TraceTask +from verifiers.v1.trace import Error, Trace, TraceTask from verifiers.v1.utils.generic import concrete_type from verifiers.v1.utils.memory import trim_memory_periodically from verifiers.v1.utils.retries import run_episode_with_retry @@ -244,7 +244,6 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -267,17 +266,6 @@ async def run_episode( ), ) agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard) - if restore is not None: - # A one-seat episode's run is its one trace: its agent's run continues it. - agent = agents.only() - if agent is None: - logger.warning( - "%s has several agent seats; running trace %s from scratch", - type(self).__name__, - restore[0].id, - ) - else: - agent.pending_restore = restore try: async with asyncio.timeout(self.config.timeout.episode): async with boundary(EnvError, f"{type(self).__name__}.setup()"): @@ -332,17 +320,13 @@ async def run_slot( semaphore: asyncio.Semaphore | None = None, on_complete: Callable[[Episode], Awaitable[None]] | None = None, on_trace: Callable[[Trace], None] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """Run one planned episode to completion, with whole-episode retries per `--env.retries`; `semaphore` bounds concurrent EPISODES — one permit for the attempt in flight, held across the whole of it (its agents, their boxes, `finalize()`) and released before a retry's backoff and before `on_complete` (the runners' persistence hook, which fires when final). - `on_trace` sees each trace at mint, after it joined `slot.traces`. `restore` - continues a trace another process ran (one-seat episodes); only the first - attempt restores it, a whole-episode retry starts over.""" - pending = [restore] if restore is not None else [] + `on_trace` sees each trace at mint, after it joined `slot.traces`.""" attempts = 0 @@ -370,7 +354,6 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, - restore=pending.pop() if pending else None, ) episode = await run_episode_with_retry(attempt, self.config.retries) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 963d63c931..0f4d019ae0 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -59,11 +59,11 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False """Whether a new process of this harness can continue a rollout in a box restored from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout - checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_CHECKPOINTS` too. Distinct + checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_SNAPSHOTS` too. Distinct from `SUPPORTS_RESUME`: a restore never finds a live process, its program continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index e2790dfbd4..3a7f2e8b70 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,7 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index e2b4495e27..d66d9904b0 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -42,7 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index e45406ab40..57f4eced3a 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -63,7 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 431c95f698..252a23118f 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,7 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 1c22e675e7..cbf76160d5 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -142,14 +142,14 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - # A retry keeps the directory: it holds `RLM_HOME`, whose session a box - # restored from a checkpoint loads. + # A retry runs the same harness and keeps the directory: it holds + # `RLM_HOME`, whose session a box restored from a checkpoint loads. if self._skills_install_dir is None: self._skills_install_dir = ( f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 96ad61ea28..bf90806e66 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -38,6 +38,8 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, + backend_cls, make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms @@ -181,17 +183,18 @@ def __init__( self._attempt = attempt self._restore: Checkpoint | None = None self._harness_data: TaskData | None = None - self.deleted_checkpoints: set[str] = set() + self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" self._pruning = asyncio.Lock() self._boxless = False - """Whether this run's checkpoints are box-less (`Checkpoint.box`).""" - self.rewound_checkpoints: list[str] = [] - """Checkpoints the restored trace dropped (taken after the one restored).""" + """Whether this run's checkpoints are box-less (no `Checkpoint.snapshot`).""" + self.rewound_snapshots: list[Snapshot] = [] + """The snapshots of the checkpoints the restored trace dropped (taken after the + one restored).""" if restore is not None: self.trace, self._restore = restore self.trace.attempt = attempt - self.rewound_checkpoints = self.trace.rewind(self._restore) + self.rewound_snapshots = self.trace.rewind(self._restore) self._restore.restored = True else: self.trace = Trace( @@ -316,8 +319,10 @@ async def open(self) -> bool: # A restored trace keeps its id; its new box must not collide with the # box an earlier attempt or process left behind. name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" - if self._restore.box: - config = config.model_copy(update={"checkpoint": self._restore.id}) + if self._restore.snapshot is not None: + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None @@ -397,7 +402,7 @@ async def open(self) -> bool: self.trace.agent.runtime = runtime.info.model_copy() # A box snapshot needs a runtime that takes them; a harness that keeps # nothing in its box is restorable from the trace alone (box-less). - self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS if ( self._checkpointing is not None and self._borrowed_runtime is None @@ -429,7 +434,7 @@ async def open(self) -> bool: self.trace.notify() # A restored box already holds the task's setup; only the harness's # (idempotent) install runs again. A box-less restore boots a fresh box. - if self._restore is None or not self._restore.box: + if self._restore is None or self._restore.snapshot is None: async with boundary(TaskError, "task setup"): await invoke( self.task.setup, @@ -813,6 +818,11 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: # box is not idle, so no snapshot would match this turn's prefix. return assert self._harness_session is not None + # Copied before the first await: the reply may commit and run its hooks while + # the harness answers, and the checkpoint must hold the state before them. + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(self._session.pinned_tool_results) try: harness = await self._harness_session.checkpoint_state() except Exception: @@ -824,20 +834,19 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: return if harness is None: return - state = copy.deepcopy(self.trace.state) - info = copy.deepcopy(self.trace.info) - pins = dict(self._session.pinned_tool_results) started = time.time() + snapshot: Snapshot | None = None if self._boxless: checkpoint_id = uuid.uuid4().hex else: try: - checkpoint_id = await self.runtime.backend.checkpoint() + snapshot = await self.runtime.backend.snapshot() except Exception: logger.warning( - "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + "snapshot failed (rollout %s)", self.trace.id, exc_info=True ) return None + checkpoint_id = snapshot.id node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None checkpoint = Checkpoint( id=checkpoint_id, @@ -847,7 +856,7 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: info=info, pins=pins, harness=harness, - box=not self._boxless, + snapshot=snapshot, time=TimeSpan(start=started), captured=time.time(), ) @@ -866,9 +875,11 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: return checkpoint async def _make_durable(self, checkpoint: Checkpoint) -> None: - assert self.runtime is not None + assert self.runtime is not None and checkpoint.snapshot is not None try: - checkpoint.bytes = await self.runtime.backend.wait_checkpoint(checkpoint.id) + checkpoint.bytes = await self.runtime.backend.wait_snapshot( + checkpoint.snapshot + ) except Exception: # Not restorable: no retry may pick it. The backend still holds it as # stray and frees it with the box. @@ -895,26 +906,27 @@ async def _prune(self) -> None: keep = kept(self.trace.checkpoints, self._checkpointing.keep) drop = [ *( - c.id + c.snapshot for c in self.trace.checkpoints - if c.box and c.time.end and not c.deleted and c.id not in keep + if c.snapshot is not None + and c.time.end + and not c.deleted + and c.id not in keep ), *( - c - for c in self.rewound_checkpoints - if c not in self.deleted_checkpoints + s + for s in self.rewound_snapshots + if s.id not in self.deleted_snapshots ), ] if drop: - self.record_deleted( - await self.runtime.backend.discard_checkpoints(drop) - ) + self.record_deleted(await self.runtime.backend.discard_snapshots(drop)) def record_deleted(self, ids: list[str]) -> None: """Mark the snapshots with these ids deleted.""" - self.deleted_checkpoints.update(ids) + self.deleted_snapshots.update(ids) for checkpoint in self.trace.checkpoints: - if checkpoint.id in self.deleted_checkpoints: + if checkpoint.id in self.deleted_snapshots: checkpoint.deleted = True async def _settle_checkpoints(self, *, wait: bool) -> None: @@ -975,15 +987,15 @@ async def abort(self) -> None: owned = self._checkpointing is None or self._checkpointing.after == "delete" with contextlib.suppress(Exception): self.record_deleted( - await self.runtime.backend.discard_checkpoints( + await self.runtime.backend.discard_snapshots( [ - *(self.rewound_checkpoints if owned else ()), + *(self.rewound_snapshots if owned else ()), *( - c.id + c.snapshot for c in self.trace.checkpoints - if owned and c.box and not c.deleted + if owned and c.snapshot is not None and not c.deleted ), - *self.runtime.backend.stray_checkpoints, + *self.runtime.backend.stray_snapshots, ] ) ) @@ -1283,9 +1295,9 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() - if runtime is not None and runtime.backend.stray_checkpoints: - await runtime.backend.discard_checkpoints( - runtime.backend.stray_checkpoints + if runtime is not None and runtime.backend.stray_snapshots: + await runtime.backend.discard_snapshots( + runtime.backend.stray_snapshots ) except Exception: logger.warning("runtime teardown failed", exc_info=True) diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 46f82c44f6..7081299e8b 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -28,6 +28,8 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, + SnapshotType, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -108,6 +110,8 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "RuntimeConfig", "RuntimeInfo", "RuntimeProcess", + "Snapshot", + "SnapshotType", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index a2e8df945c..081af6f5c9 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -28,6 +28,7 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, parse_gpu, ) from verifiers.v1.runtimes.limiters import creation_limiter @@ -154,7 +155,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = True + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -293,56 +294,59 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e - async def checkpoint(self) -> str: + async def snapshot(self) -> Snapshot: # The platform fixes the snapshot's content before the request returns, then # uploads it. Until it is durable the snapshot is stray: the platform refuses # to delete it while this box lives, so teardown frees it after. - async def create() -> str: + async def create() -> Snapshot: checkpoint = await self._client.checkpoint(self.info.id) - self.stray_checkpoints.append(checkpoint.id) - return checkpoint.id + snapshot = Snapshot(id=checkpoint.id) + self.stray_snapshots.append(snapshot) + return snapshot try: return await run_shielded(create()) except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e - async def wait_checkpoint(self, checkpoint: str) -> int | None: + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: started = time.monotonic() try: durable = await self._client.wait_for_checkpoint( - checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + snapshot.id, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS ) except Exception as e: - raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e - self.stray_checkpoints.remove(checkpoint) + raise SandboxError(f"prime checkpoint {snapshot.id}: {e}") from e + self.stray_snapshots.remove(snapshot) logger.debug( "prime: checkpoint %s durable after %.1fs on sandbox %s", - checkpoint, + snapshot.id, time.monotonic() - started, self.info.id, ) return durable.stored_bytes - async def discard_checkpoints(self, ids: list[str]) -> list[str]: + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: # The platform refuses only the newest checkpoint of a running box (the parent # of its live state); older ones go while it runs, their data kept as long as a - # newer one needs it. - from prime_sandboxes import AsyncSandboxClient - + # newer one needs it. Usually called after teardown released `_client`. deleted = [] - async with AsyncSandboxClient() as client: - for checkpoint in ids: + async with leased_client() as client: + for snapshot in snapshots: try: - await client.delete_checkpoint(checkpoint) + await client.delete_checkpoint(snapshot.id) except Exception as e: # noqa: BLE001 - provider cleanup is best-effort logger.warning( - "prime: failed to delete checkpoint %s: %s", checkpoint, e + "prime: failed to delete checkpoint %s: %s", snapshot.id, e ) else: - deleted.append(checkpoint) + deleted.append(snapshot.id) return deleted + @classmethod + def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: + return config.model_copy(update={"checkpoint": snapshot.id}) + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 7a917b6f81..f61615fe42 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,11 +13,12 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import BaseModel from pydantic_config import BaseConfig -from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType from verifiers.v1.errors import SandboxError from verifiers.v1.utils.aio import run_shielded @@ -140,6 +141,15 @@ class BaseRuntimeInfo(BaseConfig): (`Agent.run(runtime=...)`) rather than provisioning its own.""" +class Snapshot(BaseModel): + """A snapshot of a box that new boxes can boot from (`RuntimeBackend.snapshot`, + `restore_config`), as many times as needed.""" + + id: str + """The provider's id for the snapshot.""" + type: SnapshotType = "filesystem" + + @dataclass class RuntimeCache: """Framework state for one provisioned resource, kept off the backend contract: @@ -184,11 +194,10 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False - """Whether `checkpoint` snapshots the box's filesystem for a new box of this config - to boot from (its config's `checkpoint`). A rollout checkpoints before each model - turn, so a retry restores the newest turn instead of replaying the run (see - `AgentConfig.checkpoint`).""" + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False + """Whether `snapshot` captures the box's filesystem for new boxes to boot from + (`restore_config`). Rollout checkpointing (`AgentConfig.checkpoint`) takes one at + model turns, so a retry continues from there instead of starting over.""" info: BaseRuntimeInfo @@ -198,8 +207,8 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() - self.stray_checkpoints: list[str] = [] - """Checkpoints this box took that never became restorable (a failed or cancelled + self.stray_snapshots: list[Snapshot] = [] + """Snapshots this box took that never became restorable (a failed or cancelled durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] @@ -227,24 +236,30 @@ def type(self) -> str: async def start(self) -> None: pass - async def checkpoint(self) -> str: - """Snapshot the filesystem and return the checkpoint's id once its content is - fixed: writes after this returns are not in it. It becomes restorable only later - (`wait_checkpoint`); the box keeps running.""" - raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + async def snapshot(self) -> Snapshot: + """Snapshot the box and return it once its content is fixed: writes after this + returns are not in it. It becomes restorable only later (`wait_snapshot`); the + box keeps running.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") - async def wait_checkpoint(self, checkpoint: str) -> int | None: - """Wait until `checkpoint` is restorable and return its stored size in bytes, + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + """Wait until `snapshot` is restorable and return its stored size in bytes, when the provider reports it. Raises if it never becomes restorable. The box - must stay alive until then: its teardown fails a pending checkpoint.""" - raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + must stay alive until then: its teardown fails a pending snapshot.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") - async def discard_checkpoints(self, ids: list[str]) -> list[str]: - """Free checkpoints nothing will restore from and return the ids it freed. + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + """Free snapshots nothing will restore from and return the ids it freed. Best-effort: a provider may refuse one a live box still needs (prime: the newest one of a running box).""" return [] + @classmethod + def restore_config(cls, config: Any, snapshot: Snapshot) -> Any: + """The runtime config for a new box that boots from `snapshot` instead of the + task's image.""" + raise NotImplementedError(f"{cls.__name__} does not support snapshots") + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index b2de785353..f2ee570738 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.sampling import Sampling from verifiers.v1.errors import ProviderError, stop_condition from verifiers.v1.graph import RECORD_FLOAT_DECIMALS, MessageNode -from verifiers.v1.runtimes import RuntimeInfo +from verifiers.v1.runtimes import RuntimeInfo, Snapshot from verifiers.v1.semantic import ACPInfo, ParentLink, SemanticEdgeSet from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData @@ -381,12 +381,12 @@ class Checkpoint(BaseModel): turn that followed. A retry restores it (`Trace.rewind`).""" id: str - """The runtime's checkpoint id (`Runtime.checkpoint`), or this record's own for a - box-less checkpoint.""" - box: bool = True - """Whether a runtime snapshot holds the box. A box-less checkpoint (a harness that - keeps nothing in its box, on a runtime without snapshots) restores into a fresh box - whose task setup runs again; the trace and its typed state carry the rest.""" + """The snapshot's id, or this record's own for a box-less checkpoint.""" + snapshot: Snapshot | None = None + """The box's snapshot (`RuntimeBackend.checkpoint`) a retry boots a new box from. + None for a box-less checkpoint (a harness that keeps nothing in its box, on a runtime + without snapshots): it restores into a fresh box whose task setup runs again, and + the trace and its typed state carry the rest.""" node: int | None = None """The last committed node whose conversation the box's state follows; None when the snapshot preceded the first turn.""" @@ -423,56 +423,6 @@ class Checkpoint(BaseModel): can no longer be restored.""" -class PinnedResult(BaseModel): - """One pinned tool result (`RolloutSession.pinned_tool_results`), keyed by the - conversation prefix that issued its call and the call's id.""" - - prefix: str - call_id: str - message: ToolMessage - - -class RestorePoint(BaseModel): - """A checkpoint with everything a restore needs, serializable: its live-only fields - travel with it, so another process (an orchestrator resuming its run) can restore - the trace from it (`RunRequest.restore`).""" - - checkpoint: Checkpoint - tail: Messages = Field(default_factory=list) - state: dict[str, Any] | None = None - info: dict[str, Any] | None = None - pins: list[PinnedResult] = Field(default_factory=list) - - @classmethod - def of(cls, checkpoint: Checkpoint) -> RestorePoint: - state = checkpoint.state - return cls( - checkpoint=checkpoint, - tail=list(checkpoint.tail), - state=state.model_dump(mode="json") - if isinstance(state, BaseModel) - else state, - info=checkpoint.info, - pins=[ - PinnedResult(prefix=prefix, call_id=call_id, message=message) - for (prefix, call_id), message in checkpoint.pins.items() - ], - ) - - def live(self, state_type: type[BaseModel]) -> Checkpoint: - """The checkpoint with its live-only fields back, the state as `state_type`.""" - return self.checkpoint.model_copy( - update={ - "tail": list(self.tail), - "state": state_type.model_validate(self.state) - if self.state is not None - else None, - "info": self.info, - "pins": {(p.prefix, p.call_id): p.message for p in self.pins}, - } - ) - - def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -998,14 +948,6 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None - @property - def restorable(self) -> Checkpoint | None: - """The newest checkpoint a restore can use: restorable and not deleted.""" - return next( - (c for c in reversed(self.checkpoints) if c.time.end and not c.deleted), - None, - ) - @property def has_restored(self) -> bool: """Whether a retry restored this run from a checkpoint rather than starting over.""" @@ -1116,15 +1058,18 @@ def rewind_state(self, checkpoint: Checkpoint) -> None: if checkpoint.info is not None: self.info = copy.deepcopy(checkpoint.info) - def rewind(self, checkpoint: Checkpoint) -> list[str]: + def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: """Roll the graph back to `checkpoint` — the nodes committed after it go, its own conversation stays sampled — and reopen the trace for another attempt: the outcome stamps reset, while `errors` and every call stay as the attempts' history (the - calls of the dropped nodes unlinked, so usage still counts them). Returns the ids - of the later box snapshots this drops.""" + calls of the dropped nodes unlinked, so usage still counts them). Returns the + later box snapshots this drops.""" keep = 0 if checkpoint.node is None else checkpoint.node + 1 if keep > len(self.nodes): raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + ids = [c.id for c in self.checkpoints] + if checkpoint.id not in ids: + raise ValueError(f"checkpoint {checkpoint.id!r} is not on this trace") del self.nodes[keep:] for node in self.nodes: node.semantic_parents = [ @@ -1139,9 +1084,12 @@ def rewind(self, checkpoint: Checkpoint) -> list[str]: self.rewind_state(checkpoint) # The restored rollout re-stamps every stage span; the run's start stays. self.timing = Timing(start=self.timing.start) - rewound = [c for c in self.checkpoints if c.node is not None and c.node >= keep] - self.checkpoints = [c for c in self.checkpoints if c not in rewound] - dropped = [c.id for c in rewound if c.box] + # Checkpoints are in the order they were taken: the ones after `checkpoint` hold + # the future it rewinds past, including those at its own node. + position = ids.index(checkpoint.id) + 1 + rewound = self.checkpoints[position:] + del self.checkpoints[position:] + dropped = [c.snapshot for c in rewound if c.snapshot is not None] self.rewards = {} self.metrics = {} self.dropped = None From 7a5330a2c6093f598c1c125d1331d3212e266560 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 23:15:04 +0000 Subject: [PATCH 13/53] record cancelled turns and shorten checkpoint docs A turn cancelled while its reply waits for the snapshot capture now still records its call and abandons its preview. The wait is shielded, so the snapshot finishes in the background, where the rollout awaits it. A capture reads the turn's node and tail before its first await, like the state copies, so everything a checkpoint describes comes from the idle box. Docs: show the checkpoint config as a TOML example, and shorten the harness flag's entry. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 10 ++++- docs/harnesses.md | 2 +- verifiers/v1/interception/server.py | 66 ++++++++++++++++------------- verifiers/v1/rollout.py | 9 ++-- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 4b775c23c1..fc5f8c0a34 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,4 +77,12 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box at model turns, and a retry restores the newest snapshot and continues the trace from there. Only retries within one `Agent.run` restore; a new process starts over. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_SNAPSHOTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `type` sets what a snapshot captures (`"filesystem"`, the only type today), `interval` how often it snapshots (every 10th turn by default), `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Set `--no-env.agent.checkpoint` to turn it off. + +```toml +[env.agent.checkpoint] +type = "filesystem" # what a snapshot captures +interval = 10 # snapshot every 10th model turn +keep = "last" # snapshots kept while the rollout runs: "last", "all", or every k-th +after = "delete" # when the run is final: "delete" the kept snapshots, or "keep" them +``` diff --git a/docs/harnesses.md b/docs/harnesses.md index 2144e499ff..624e909037 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem snapshot, so a retry restores the newest snapshot instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. +- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that a new process of the harness can continue a rollout in a box restored from a filesystem snapshot (`agent.checkpoint`). Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index a82a177bc4..f891951ebc 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1046,36 +1046,42 @@ async def sample() -> web.Response: error = e raise finally: - if capture is not None: - held = time.monotonic() - try: - checkpoint = await capture - # A lost snapshot never fails the turn. - except Exception: - logger.warning( - "checkpoint failed: id=%s", session.trace.id, exc_info=True - ) - else: - if checkpoint is not None: - checkpoint.held = time.monotonic() - held - if node is None: - turn.abandon() - # The turn's one per-exchange record: settings, timing, outcome, and - # the error that ended it (if any). - self.record_call( - session, - dialect, - body, - started, - node=node, - finish_reason=call_response.finish_reason - if call_response - else None, - usage=call_response.usage if call_response else None, - error=error, - policy_paths=policy_paths, - acp=acp, - ) + try: + if capture is not None: + held = time.monotonic() + try: + # Shielded: a cancelled exchange leaves the snapshot to + # finish in the background, where the rollout awaits it. + checkpoint = await asyncio.shield(capture) + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", + session.trace.id, + exc_info=True, + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held + finally: + if node is None: + turn.abandon() + # The turn's one per-exchange record: settings, timing, outcome, + # and the error that ended it (if any). + self.record_call( + session, + dialect, + body, + started, + node=node, + finish_reason=call_response.finish_reason + if call_response + else None, + usage=call_response.usage if call_response else None, + error=error, + policy_paths=policy_paths, + acp=acp, + ) return serve(call_response, events) if streaming: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index bf90806e66..4ec0f307c4 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -818,8 +818,10 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: # box is not idle, so no snapshot would match this turn's prefix. return assert self._harness_session is not None - # Copied before the first await: the reply may commit and run its hooks while - # the harness answers, and the checkpoint must hold the state before them. + # Read before the first await: the reply may commit and run its hooks while + # the harness answers, and the checkpoint must describe the idle box. + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + tail = list(turn.tail) state = copy.deepcopy(self.trace.state) info = copy.deepcopy(self.trace.info) pins = dict(self._session.pinned_tool_results) @@ -847,11 +849,10 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: ) return None checkpoint_id = snapshot.id - node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None checkpoint = Checkpoint( id=checkpoint_id, node=node, - tail=list(turn.tail), + tail=tail, state=state, info=info, pins=pins, From 03a52bd89f9d5bcb859b8cd95685478f34914659 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:06:58 +0000 Subject: [PATCH 14/53] add the checkpoint restore integration test back Restore a rollout from a snapshot end to end on prime: per harness, at the default interval, for rlm with skills, and for each retention policy. The cases run once per SnapshotType, on every harness and runtime that declare SUPPORTS__SNAPSHOTS, so a new snapshot type gets its cases without a test change. CI runs it as "Checkpoint Restore". Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 2 + .../fixtures/checkpoint_restore_v1.py | 133 +++++++++++++ tests/integration/fixtures/pyproject.toml | 1 + tests/integration/grid.py | 19 +- tests/integration/test_checkpoint.py | 186 ++++++++++++++++++ uv.lock | 9 +- 6 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 tests/integration/fixtures/checkpoint_restore_v1.py create mode 100644 tests/integration/test_checkpoint.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 067a4883e7..19409ffe18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,6 +97,8 @@ jobs: tests: tests/integration/test_user_echo.py - behavior: Resume Tool tests: tests/integration/test_resume_tool.py + - behavior: Checkpoint Restore + tests: tests/integration/test_checkpoint.py - behavior: User Sim tests: tests/integration/test_user_sim.py - behavior: Agent User diff --git a/tests/integration/fixtures/checkpoint_restore_v1.py b/tests/integration/fixtures/checkpoint_restore_v1.py new file mode 100644 index 0000000000..addfaf2b2e --- /dev/null +++ b/tests/integration/fixtures/checkpoint_restore_v1.py @@ -0,0 +1,133 @@ +"""checkpoint-restore: an agentic task whose first attempt dies mid-rollout, so the retry +restores the box's newest snapshot instead of starting over. + +`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model +for two tool calls that build on each other (the second needs the first's token, so they +take two turns), and the reward reads the second one's file back out of the box. The +tools are colocated: they run in the box, so their files are in its snapshots, and any +harness with MCP tools can drive them. A request `@intercept` raises a `ProviderError` +once the first attempt has made two turns — after the snapshot of turn 2 exists. The +restored attempt continues from that snapshot in a new harness process: it never sees +the injected failure again (it fires once per task), finds the earlier calls' files, and +finishes. Needs a harness and a runtime that both support the snapshot type +(`SUPPORTS__SNAPSHOTS`; today prime's filesystem snapshots). +""" + +import shutil +import uuid +from pathlib import Path + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +MARKER = "marker.txt" +STEP_ONE = "step1.txt" +TOKEN = "token.txt" +DONE = "done.txt" +FAIL_AFTER_TURNS = 2 + + +class RestoreState(vf.State): + workdir: str = "" + """The box's working directory, where `setup` put the marker.""" + + +class StepsToolset(vf.Toolset[vf.ToolsetConfig, RestoreState]): + TOOL_PREFIX = "steps" + + @vf.tool + def copy_marker(self) -> str: + """Copy the marker file to the step-one file. Returns the token `write_done` + needs.""" + workdir = Path(self.state.workdir) + shutil.copyfile(workdir / MARKER, workdir / STEP_ONE) + token = uuid.uuid4().hex[:8] + (workdir / TOKEN).write_text(token) + return f"copied; token={token}" + + @vf.tool + def write_done(self, token: str) -> str: + """Write the step-one file's content to the done file. Needs the token that + `copy_marker` returned.""" + workdir = Path(self.state.workdir) + if not (workdir / TOKEN).is_file() or token != (workdir / TOKEN).read_text(): + return "wrong token: call copy_marker first and pass the token it returns" + shutil.copyfile(workdir / STEP_ONE, workdir / DONE) + return "written" + + +class CheckpointRestoreTaskConfig(vf.TaskConfig): + tools: vf.ToolsetConfig = vf.ToolsetConfig(colocated=True) + + +class CheckpointRestoreConfig(vf.TasksetConfig): + task: CheckpointRestoreTaskConfig = CheckpointRestoreTaskConfig() + + +class CheckpointRestoreTask( + vf.Task[vf.TaskData, RestoreState, CheckpointRestoreTaskConfig] +): + failed = False + """Whether the injected failure fired: once per task, so a retry from scratch + (checkpoints off) also gets through.""" + + @classmethod + def toolsets(cls, config: CheckpointRestoreTaskConfig) -> list[vf.Toolset]: + return [StepsToolset(config.tools)] + + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.write(MARKER, nonce.encode()) + result = await runtime.run(["pwd"], {}) + trace.state.workdir = result.stdout.strip() + + @vf.intercept + def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: + # Request hooks also judge each proposed tool call after a turn commits, over + # the branch plus an empty probe result; fail only as a real turn goes upstream + # (every tool returns text, so a real result is never empty). + last = request.messages[-1] + probe = isinstance(last, vf.ToolMessage) and not last.content + if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not self.failed: + self.failed = True + # A 4xx: harness SDKs end the turn at once instead of retrying it until the + # rollout's deadline. + raise vf.ProviderError("injected mid-rollout failure", status_code=400) + + @vf.reward(weight=1.0) + async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: + try: + content = (await runtime.read(DONE)).decode(errors="replace") + except (SandboxError, OSError, ValueError): + return 0.0 + return float(content.strip() == trace.info["nonce"]) + + +class CheckpointRestoreTaskset( + vf.Taskset[CheckpointRestoreTask, CheckpointRestoreConfig] +): + def load(self) -> list[CheckpointRestoreTask]: + return [ + CheckpointRestoreTask( + vf.TaskData( + idx=0, + prompt=( + "Use the tools of the `steps` MCP server. Make one tool call " + "per turn and wait for each result before the next.\n" + "1. Call `copy_marker`.\n" + "2. Call `write_done` with the token `copy_marker` returned.\n" + "3. Reply with the single word: done." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["CheckpointRestoreTaskset"] + + +if __name__ == "__main__": + StepsToolset.run() diff --git a/tests/integration/fixtures/pyproject.toml b/tests/integration/fixtures/pyproject.toml index 917d9ba656..675c238d8b 100644 --- a/tests/integration/fixtures/pyproject.toml +++ b/tests/integration/fixtures/pyproject.toml @@ -15,6 +15,7 @@ build-backend = "hatchling.build" [tool.hatch.build] include = [ "agent_user.py", + "checkpoint_restore_v1.py", "counter_tool.py", "echo_acp_resume.py", "echo_tool.py", diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 2d636ccd32..db0fd9d34e 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,7 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). +`SUPPORTS_RESUME` or a native session). A `-snapshots` need (one per +`SnapshotType`) asks both the harness and the runtime for `SUPPORTS__SNAPSHOTS`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -34,7 +35,7 @@ from pydantic import TypeAdapter import verifiers.v1 as vf -from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType from verifiers.v1.harness import Harness from verifiers.v1.runtimes import PrimeBackend, RuntimeConfig, backend_cls from verifiers.v1.utils.loaders import harness_class, harness_config_type @@ -65,8 +66,15 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", + **{f"{kind}-snapshots": f"{kind} snapshots" for kind in get_args(SnapshotType)}, } + +def snapshot_flag(kind: str) -> str: + """The flag a harness and a runtime both set for snapshots of `kind`.""" + return f"SUPPORTS_{kind.upper()}_SNAPSHOTS" + + # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -139,6 +147,9 @@ def supports(runtime: str, feature: str) -> bool: return isinstance(config, NetworkPolicyConfig) if feature == "services": return issubclass(runtime_cls(runtime), PrimeBackend) + if feature.endswith("-snapshots"): + flag = snapshot_flag(feature.removesuffix("-snapshots")) + return getattr(runtime_cls(runtime), flag, False) flag = { "container": "SUPPORTS_TASK_IMAGES", "user": "SUPPORTS_USER_SWITCHING", @@ -173,6 +184,10 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" + for need in needs: + kind = need.removesuffix("-snapshots") + if kind != need and not getattr(cls, snapshot_flag(kind), False): + return f"{name} cannot restore {kind} snapshots" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py new file mode 100644 index 0000000000..85a82e4a8e --- /dev/null +++ b/tests/integration/test_checkpoint.py @@ -0,0 +1,186 @@ +"""A retry restores the box's newest snapshot (checkpoint-restore-v1: the first attempt +dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns +before it, and finishes in a new harness process). + +Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime +that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. +A new snapshot type adds its cases without a change here. The retention policies run +on one harness: what they keep or delete does not depend on it.""" + +from collections.abc import Sequence +from typing import get_args + +import pytest +from checkpoint_restore_v1 import FAIL_AFTER_TURNS +from grid import grid +from prime_sandboxes import AsyncSandboxClient +from skill_codeword import SKILL + +from verifiers.v1.configs.runtime import SnapshotType +from verifiers.v1.harness import RESTORE_PROMPT +from verifiers.v1.rollout import kept +from verifiers.v1.types import content_text + +RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} + + +def snapshot_grid( + harnesses: Sequence[str] | None = None, needs: tuple[str, ...] = ("mcp",) +) -> list: + """`(snapshot_type, harness, harness_runtime)` cells: the harness grid once per + snapshot type, each cell needing that type's snapshots.""" + return [ + pytest.param(kind, *cell.values, marks=cell.marks, id=f"{kind}-{cell.id}") + for kind in get_args(SnapshotType) + for cell in grid( + *([harnesses] if harnesses else []), needs=(f"{kind}-snapshots", *needs) + ) + ] + + +def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: + return { + "agent": { + "retries": RETRIES, + "checkpoint": {"type": snapshot_type, **checkpoint}, + } + } + + +async def on_platform(checkpoint: str) -> bool: + """Whether prime still holds `checkpoint`, restorable.""" + async with AsyncSandboxClient() as client: + try: + return (await client.get_checkpoint(checkpoint)).state == "DURABLE" + except Exception: # noqa: BLE001 - a deleted checkpoint is not found + return False + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(), + indirect=["harness", "harness_runtime"], +) +async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_runtime): + """The box restores to the state before the turn that failed, the turns up to + there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not + rerun, and the harness is told about the restore — the restored attempt builds on + the earlier tool call's file. A snapshot every turn, so the failure at turn 3 + restores a snapshot with turns behind it.""" + env = checkpoint_env(snapshot_type, interval=1) + trace = await restored_trace(run, harness, harness_runtime, env) + assert trace.num_turns >= FAIL_AFTER_TURNS + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["bash"]), + indirect=["harness", "harness_runtime"], +) +async def test_default_config_restores(run, snapshot_type, harness, harness_runtime): + """With the default interval (every 10th turn), the retry restores the turn-1 + snapshot: the box as `setup` left it, before any tool call.""" + env = checkpoint_env(snapshot_type) + trace = await restored_trace(run, harness, harness_runtime, env) + assert len(trace.checkpoints) >= 1 + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["rlm"], needs=("mcp", "skills")), + indirect=["harness", "harness_runtime"], +) +async def test_rlm_with_skills_restores(run, snapshot_type, harness, harness_runtime): + """rlm keeps its session in a per-run skill home when it has skills: the retry + must load the session from the same home the restored box holds.""" + config = harness.model_copy(update={"skills": [SKILL]}) + env = checkpoint_env(snapshot_type, interval=1) + trace = await restored_trace(run, config, harness_runtime, env) + assert trace.num_turns >= FAIL_AFTER_TURNS + + +async def restored_trace(run, harness, harness_runtime, env): + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=env, + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + # The newest snapshot before the failed turn boots the retry's box. + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert restored_from is not None + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + assert trace.has_restored + assert trace.attempt == 2 + # The error belongs to the first attempt, in the agent stage. + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] + # The restored process made no side request of its own (it would start a second + # root), e.g. a model probe on session load. + assert sum(node.parent is None for node in trace.nodes) == 1 + # The restored process got the restore notice as a user turn. + assert any( + node.message.role == "user" + and RESTORE_PROMPT in content_text(node.message.content) + for node in trace.nodes + ) + # Every turn kept has its call; the dropped turns' calls stay, unlinked, so usage + # counts them. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace + + +@pytest.mark.parametrize( + "keep,after", + [("last", "delete"), ("last", "keep"), ("all", "keep"), (2, "keep")], + ids=["keep-last-delete", "keep-last", "keep-all", "keep-every-2nd"], +) +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["bash"]), + indirect=["harness", "harness_runtime"], +) +async def test_checkpoint_retention( + run, snapshot_type, harness, harness_runtime, keep, after +): + """Snapshots the policy drops are deleted while the rollout runs, without breaking + the newest one: the retry still restores it. When the run is final, `after` deletes + the rest or leaves exactly what `keep` holds.""" + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(snapshot_type, interval=1, keep=keep, after=after), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert trace.has_restored + live = [c.id for c in trace.checkpoints if not c.deleted] + try: + if after == "delete": + assert live == [] + else: + assert set(live) == kept(trace.checkpoints, keep) + if keep == "last": + assert len(live) == 1 + if keep == "all": + # Snapshots still uploading when the run succeeded were never restorable. + assert live == [c.id for c in trace.checkpoints if c.time.end] + for checkpoint in trace.checkpoints: + assert await on_platform(checkpoint.id) == (not checkpoint.deleted) + finally: + async with AsyncSandboxClient() as client: + for checkpoint in live: + await client.delete_checkpoint(checkpoint) diff --git a/uv.lock b/uv.lock index 559e7ac5cd..f52a3a2e23 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 4 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -31,6 +31,12 @@ renderers = false ruff = "2026-07-28T00:00:00Z" ty = "2026-07-28T00:00:00Z" +[manifest] +constraints = [ + { name = "brotli", specifier = ">=1.2.1" }, + { name = "brotlicffi", specifier = ">=1.2.1.0" }, +] + [[package]] name = "aiofile" version = "3.12.3" @@ -5067,7 +5073,6 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } -default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, From 94cf961682298c37eec17099916688f9bc117624 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:07:06 +0000 Subject: [PATCH 15/53] keep uv.lock as on the branch Co-Authored-By: Claude Opus 5.5 --- uv.lock | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/uv.lock b/uv.lock index f52a3a2e23..559e7ac5cd 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 4 +revision = 5 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -31,12 +31,6 @@ renderers = false ruff = "2026-07-28T00:00:00Z" ty = "2026-07-28T00:00:00Z" -[manifest] -constraints = [ - { name = "brotli", specifier = ">=1.2.1" }, - { name = "brotlicffi", specifier = ">=1.2.1.0" }, -] - [[package]] name = "aiofile" version = "3.12.3" @@ -5073,6 +5067,7 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } +default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, From e04209188ea0e0e051983b628cbd361a252504c9 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:17:34 +0000 Subject: [PATCH 16/53] drop box-less checkpoints A box-less checkpoint let a harness that runs no code restore from the trace alone on a runtime without snapshots. It served restores in another process, which this PR no longer does; for retries in one run it only saves the few turns of a short chat rollout. Every checkpoint now has a snapshot, and the boxless setting is gone. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_checkpoint.py | 31 ++++++------------ verifiers/v1/agent.py | 10 +++--- verifiers/v1/configs/agent.py | 3 -- verifiers/v1/rollout.py | 59 +++++++++++------------------------ verifiers/v1/trace.py | 11 +++---- 5 files changed, 37 insertions(+), 77 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 414c877189..004a086e22 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -50,7 +50,9 @@ def make_trace() -> vf.Trace: def test_rewind_restores_info_with_state(): trace = make_trace() - checkpoint = Checkpoint(id="ck", node=1, info={"setup": "nonce"}) + checkpoint = Checkpoint( + id="ck", snapshot=vf.Snapshot(id="ck"), node=1, info={"setup": "nonce"} + ) trace.checkpoints.append(checkpoint) trace.info = {"setup": "nonce", "scoring_skipped": "stale"} trace.rewind(checkpoint) @@ -59,7 +61,7 @@ def test_rewind_restores_info_with_state(): def test_rewind_keeps_the_dropped_turns_calls(): trace = make_trace() - checkpoint = Checkpoint(id="ck", node=0) + checkpoint = Checkpoint(id="ck", snapshot=vf.Snapshot(id="ck"), node=0) trace.checkpoints.append(checkpoint) trace.rewind(checkpoint) assert len(trace.nodes) == 1 @@ -90,8 +92,8 @@ def test_agent_time_splits_off_the_checkpoint_hold(): trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) trace.checkpoints += [ - Checkpoint(id="old", captured=90.0, held=3.0), - Checkpoint(id="new", captured=106.0, held=1.5), + Checkpoint(id="old", snapshot=vf.Snapshot(id="old"), captured=90.0, held=3.0), + Checkpoint(id="new", snapshot=vf.Snapshot(id="new"), captured=106.0, held=1.5), ] trace.split_agent_time() assert trace.timing.agent.checkpoint.duration == 1.5 @@ -173,7 +175,6 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] - run._boxless = False return run, backend @@ -247,7 +248,10 @@ def test_keep_policies(): assert kept(checkpoints, 2) == {"a", "c", "e"} assert kept(checkpoints, 3) == {"a", "d", "e"} # Deleted and not yet durable snapshots are never kept. - pending = [*durable("a", "b", deleted=("a",)), Checkpoint(id="c")] + pending = [ + *durable("a", "b", deleted=("a",)), + Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), + ] assert kept(pending, "all") == {"b"} @@ -263,15 +267,6 @@ async def test_a_durable_snapshot_prunes_the_older_ones(): assert run.deleted_snapshots == {"old1", "old2", "rewound"} -async def test_a_boxless_checkpoint_is_restorable_at_once(): - run, backend = make_rollout() - run._boxless = True - checkpoint = await run._checkpoint(TURN) - assert checkpoint is not None and checkpoint.snapshot is None - assert checkpoint.time.end and backend.taken == 0 - assert run.trace.checkpoints[-1] is checkpoint - - def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: return vf.Agent( vf.AgentConfig( @@ -304,9 +299,3 @@ def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): "a failed rollout retries from the start" ), ] - - -def test_boxless_checkpointing_is_opt_in(): - assert gated("null", vf.SubprocessConfig())._checkpointing() is None - assert gated("null", vf.SubprocessConfig(), boxless=True)._checkpointing() - assert gated("bash", vf.SubprocessConfig(), boxless=True)._checkpointing() is None diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 5fdab5eacc..6ca01e0d3e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -654,9 +654,9 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if harness.SUPPORTS_FILESYSTEM_SNAPSHOTS and ( - backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - or (config.boxless and not harness.EXECUTES_CODE) + if ( + harness.SUPPORTS_FILESYSTEM_SNAPSHOTS + and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS ): return config key = (harness.config.id, self.config.runtime.type) @@ -760,9 +760,7 @@ async def run( ) trace = run.trace snapshots.update((s.id, s) for s in run.rewound_snapshots) - snapshots.update( - (c.snapshot.id, c.snapshot) for c in trace.checkpoints if c.snapshot - ) + snapshots.update((c.snapshot.id, c.snapshot) for c in trace.checkpoints) deleted.update(run.deleted_snapshots) if target is not None and any( error.stage == "setup" for error in trace.errors[seen:] diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 833262f827..090456a3dc 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -57,9 +57,6 @@ class RolloutCheckpointConfig(BaseConfig): after: Literal["delete", "keep"] = "delete" """What happens to the kept snapshots once the run is final: `"delete"` them, or `"keep"` them for their owner to delete.""" - boxless: bool = False - """Also checkpoint a harness that runs no code (null) on a runtime without - snapshots, from the trace alone.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 4ec0f307c4..6f911a67bf 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -186,8 +186,6 @@ def __init__( self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" self._pruning = asyncio.Lock() - self._boxless = False - """Whether this run's checkpoints are box-less (no `Checkpoint.snapshot`).""" self.rewound_snapshots: list[Snapshot] = [] """The snapshots of the checkpoints the restored trace dropped (taken after the one restored).""" @@ -319,10 +317,9 @@ async def open(self) -> bool: # A restored trace keeps its id; its new box must not collide with the # box an earlier attempt or process left behind. name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" - if self._restore.snapshot is not None: - config = backend_cls(config).restore_config( - config, self._restore.snapshot - ) + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None @@ -400,13 +397,10 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() - # A box snapshot needs a runtime that takes them; a harness that keeps - # nothing in its box is restorable from the trace alone (box-less). - self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS if ( self._checkpointing is not None and self._borrowed_runtime is None - and (not self._boxless or not self.harness.EXECUTES_CODE) + and runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS ): self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: @@ -433,8 +427,8 @@ async def open(self) -> bool: self.trace.timing.setup.start = now self.trace.notify() # A restored box already holds the task's setup; only the harness's - # (idempotent) install runs again. A box-less restore boots a fresh box. - if self._restore is None or self._restore.snapshot is None: + # (idempotent) install runs again. + if self._restore is None: async with boundary(TaskError, "task setup"): await invoke( self.task.setup, @@ -837,20 +831,13 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: if harness is None: return started = time.time() - snapshot: Snapshot | None = None - if self._boxless: - checkpoint_id = uuid.uuid4().hex - else: - try: - snapshot = await self.runtime.backend.snapshot() - except Exception: - logger.warning( - "snapshot failed (rollout %s)", self.trace.id, exc_info=True - ) - return None - checkpoint_id = snapshot.id + try: + snapshot = await self.runtime.backend.snapshot() + except Exception: + logger.warning("snapshot failed (rollout %s)", self.trace.id, exc_info=True) + return None checkpoint = Checkpoint( - id=checkpoint_id, + id=snapshot.id, node=node, tail=tail, state=state, @@ -862,21 +849,16 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: captured=time.time(), ) self.trace.checkpoints.append(checkpoint) - if self._boxless: - # Nothing to upload: restorable as soon as it is recorded. - checkpoint.time.end = checkpoint.captured - self.trace.notify() - else: - self._durable[checkpoint_id] = asyncio.create_task( - self._make_durable(checkpoint) - ) + self._durable[checkpoint.id] = asyncio.create_task( + self._make_durable(checkpoint) + ) logger.debug( - "checkpoint %s taken: id=%s node=%s", checkpoint_id, self.trace.id, node + "checkpoint %s taken: id=%s node=%s", checkpoint.id, self.trace.id, node ) return checkpoint async def _make_durable(self, checkpoint: Checkpoint) -> None: - assert self.runtime is not None and checkpoint.snapshot is not None + assert self.runtime is not None try: checkpoint.bytes = await self.runtime.backend.wait_snapshot( checkpoint.snapshot @@ -909,10 +891,7 @@ async def _prune(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if c.snapshot is not None - and c.time.end - and not c.deleted - and c.id not in keep + if c.time.end and not c.deleted and c.id not in keep ), *( s @@ -994,7 +973,7 @@ async def abort(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if owned and c.snapshot is not None and not c.deleted + if owned and not c.deleted ), *self.runtime.backend.stray_snapshots, ] diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index f2ee570738..30e072b4b7 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -381,12 +381,9 @@ class Checkpoint(BaseModel): turn that followed. A retry restores it (`Trace.rewind`).""" id: str - """The snapshot's id, or this record's own for a box-less checkpoint.""" - snapshot: Snapshot | None = None - """The box's snapshot (`RuntimeBackend.checkpoint`) a retry boots a new box from. - None for a box-less checkpoint (a harness that keeps nothing in its box, on a runtime - without snapshots): it restores into a fresh box whose task setup runs again, and - the trace and its typed state carry the rest.""" + """The snapshot's id.""" + snapshot: Snapshot + """The box's snapshot (`RuntimeBackend.snapshot`) a retry boots a new box from.""" node: int | None = None """The last committed node whose conversation the box's state follows; None when the snapshot preceded the first turn.""" @@ -1089,7 +1086,7 @@ def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: position = ids.index(checkpoint.id) + 1 rewound = self.checkpoints[position:] del self.checkpoints[position:] - dropped = [c.snapshot for c in rewound if c.snapshot is not None] + dropped = [c.snapshot for c in rewound] self.rewards = {} self.metrics = {} self.dropped = None From fcf1800523f7f99c05eb4079e6f6bc02fdb3bf2e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:19:27 +0000 Subject: [PATCH 17/53] check that a checkpointed run leaks no snapshot Record every snapshot the run's boxes take, across attempts, and check each on the platform once the run is done: deleted, except the ones the retention policy keeps, which stay restorable. Before, only the checkpoints left on the final trace were checked, so a rewound snapshot or one still uploading at the end could leak unnoticed. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_checkpoint.py | 67 +++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index 85a82e4a8e..c9fce04d80 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -5,7 +5,9 @@ Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. A new snapshot type adds its cases without a change here. The retention policies run -on one harness: what they keep or delete does not depend on it.""" +on one harness: what they keep or delete does not depend on it. Every case checks that +the run leaks no snapshot: each one its boxes took is deleted on the platform, except +the ones the policy keeps.""" from collections.abc import Sequence from typing import get_args @@ -19,6 +21,7 @@ from verifiers.v1.configs.runtime import SnapshotType from verifiers.v1.harness import RESTORE_PROMPT from verifiers.v1.rollout import kept +from verifiers.v1.runtimes import PrimeBackend from verifiers.v1.types import content_text RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} @@ -47,13 +50,45 @@ def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: } -async def on_platform(checkpoint: str) -> bool: - """Whether prime still holds `checkpoint`, restorable.""" +@pytest.fixture +def taken(monkeypatch) -> list[str]: + """The ids of every snapshot the run's boxes take, in every attempt: also the + ones a restore rewinds past and the ones that never become restorable.""" + ids: list[str] = [] + snapshot = PrimeBackend.snapshot + + async def recording(self): + result = await snapshot(self) + ids.append(result.id) + return result + + monkeypatch.setattr(PrimeBackend, "snapshot", recording) + return ids + + +async def platform_state(snapshot_id: str) -> str | None: + """The snapshot's state on prime, or None once it is gone.""" async with AsyncSandboxClient() as client: try: - return (await client.get_checkpoint(checkpoint)).state == "DURABLE" - except Exception: # noqa: BLE001 - a deleted checkpoint is not found - return False + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None + + +async def assert_no_leaks(taken: list[str], kept_ids: set[str] = frozenset()) -> None: + """Every snapshot the run took is deleted, except `kept_ids`, which stay + restorable.""" + assert taken + states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} + leaked = { + snapshot_id: state + for snapshot_id, state in states.items() + if snapshot_id not in kept_ids and state not in (None, "DELETING") + } + assert leaked == {} + assert { + snapshot_id: states[snapshot_id] for snapshot_id in kept_ids + } == dict.fromkeys(kept_ids, "DURABLE") @pytest.mark.parametrize( @@ -61,7 +96,9 @@ async def on_platform(checkpoint: str) -> bool: snapshot_grid(), indirect=["harness", "harness_runtime"], ) -async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_runtime): +async def test_retry_restores_checkpoint( + run, taken, snapshot_type, harness, harness_runtime +): """The box restores to the state before the turn that failed, the turns up to there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not rerun, and the harness is told about the restore — the restored attempt builds on @@ -70,6 +107,7 @@ async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_ru env = checkpoint_env(snapshot_type, interval=1) trace = await restored_trace(run, harness, harness_runtime, env) assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) @pytest.mark.parametrize( @@ -77,12 +115,15 @@ async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_ru snapshot_grid(["bash"]), indirect=["harness", "harness_runtime"], ) -async def test_default_config_restores(run, snapshot_type, harness, harness_runtime): +async def test_default_config_restores( + run, taken, snapshot_type, harness, harness_runtime +): """With the default interval (every 10th turn), the retry restores the turn-1 snapshot: the box as `setup` left it, before any tool call.""" env = checkpoint_env(snapshot_type) trace = await restored_trace(run, harness, harness_runtime, env) assert len(trace.checkpoints) >= 1 + await assert_no_leaks(taken) @pytest.mark.parametrize( @@ -90,13 +131,16 @@ async def test_default_config_restores(run, snapshot_type, harness, harness_runt snapshot_grid(["rlm"], needs=("mcp", "skills")), indirect=["harness", "harness_runtime"], ) -async def test_rlm_with_skills_restores(run, snapshot_type, harness, harness_runtime): +async def test_rlm_with_skills_restores( + run, taken, snapshot_type, harness, harness_runtime +): """rlm keeps its session in a per-run skill home when it has skills: the retry must load the session from the same home the restored box holds.""" config = harness.model_copy(update={"skills": [SKILL]}) env = checkpoint_env(snapshot_type, interval=1) trace = await restored_trace(run, config, harness_runtime, env) assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) async def restored_trace(run, harness, harness_runtime, env): @@ -150,7 +194,7 @@ async def restored_trace(run, harness, harness_runtime, env): indirect=["harness", "harness_runtime"], ) async def test_checkpoint_retention( - run, snapshot_type, harness, harness_runtime, keep, after + run, taken, snapshot_type, harness, harness_runtime, keep, after ): """Snapshots the policy drops are deleted while the rollout runs, without breaking the newest one: the retry still restores it. When the run is final, `after` deletes @@ -178,8 +222,7 @@ async def test_checkpoint_retention( if keep == "all": # Snapshots still uploading when the run succeeded were never restorable. assert live == [c.id for c in trace.checkpoints if c.time.end] - for checkpoint in trace.checkpoints: - assert await on_platform(checkpoint.id) == (not checkpoint.deleted) + await assert_no_leaks(taken, set(live)) finally: async with AsyncSandboxClient() as client: for checkpoint in live: From d77caf0fbeaa910b9e8c345e96792f5ff1b4322f Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 02:49:12 +0000 Subject: [PATCH 18/53] join a cancelled harness turn through cancellation When a rollout is cancelled (an agent timeout, shutdown), its harness turn is cancelled and then joined. A second cancellation during that join left the turn running while teardown began, where it could still write the trace or use the box. Join it shielded, then re-raise. Co-Authored-By: Claude Opus 5.5 --- verifiers/v1/rollout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index afbeead92e..76213f9a57 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -48,6 +48,7 @@ from verifiers.v1.task import Task, TaskData from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage +from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke from verifiers.v1.utils.format import format_time @@ -781,7 +782,9 @@ async def _turn(self, messages: Messages | None) -> bool: aborted.cancel() if not turn.done(): turn.cancel() - await asyncio.wait({turn}) + # Joined even through another cancellation: a turn left running + # could still write the trace or use the box during teardown. + await run_shielded(asyncio.wait({turn})) if self._session.aborted is not None: if not turn.cancelled(): turn.exception() # the abort decides the outcome, whatever the turn did From 3a581f8cf4c484d55317616ce320988ab2c173e7 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Mon, 9 Nov 2026 23:39:05 +0000 Subject: [PATCH 19/53] snapshot from turn n and drop after, all and type From review: keep the checkpoint config to what a user decides. - interval counts from turn n (the n-th, 2n-th, ...), so a rollout shorter than n turns takes no snapshot; the turn a restore resumes at is not snapshotted again, so keep = "last" no longer deletes the snapshot that just restored - every snapshot is deleted once the run is final: drop `after` - drop keep = "all", which keep = 1 already expresses - drop `type` from the config and SnapshotType from the API: the runtime takes the best snapshot it supports and records its type on the checkpoint (Snapshot.type) Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 8 +- tests/integration/grid.py | 23 ++-- tests/integration/test_checkpoint.py | 172 +++++++++------------------ tests/unit/test_checkpoint.py | 18 ++- verifiers/v1/__init__.py | 3 - verifiers/v1/agent.py | 15 +-- verifiers/v1/configs/agent.py | 28 ++--- verifiers/v1/configs/runtime.py | 4 - verifiers/v1/rollout.py | 29 +++-- verifiers/v1/runtimes/__init__.py | 2 - verifiers/v1/runtimes/base.py | 8 +- verifiers/v1/trace.py | 4 +- 12 files changed, 117 insertions(+), 197 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index fc5f8c0a34..6c4f4cf0ca 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,12 +77,10 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] -type = "filesystem" # what a snapshot captures -interval = 10 # snapshot every 10th model turn -keep = "last" # snapshots kept while the rollout runs: "last", "all", or every k-th -after = "delete" # when the run is final: "delete" the kept snapshots, or "keep" them +interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none +keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` diff --git a/tests/integration/grid.py b/tests/integration/grid.py index db0fd9d34e..12e8a07a91 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,8 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). A `-snapshots` need (one per -`SnapshotType`) asks both the harness and the runtime for `SUPPORTS__SNAPSHOTS`. +`SUPPORTS_RESUME` or a native session). A `snapshots` need asks both the harness and +the runtime for `SUPPORTS_FILESYSTEM_SNAPSHOTS`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -35,7 +35,7 @@ from pydantic import TypeAdapter import verifiers.v1 as vf -from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType +from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import PrimeBackend, RuntimeConfig, backend_cls from verifiers.v1.utils.loaders import harness_class, harness_config_type @@ -66,15 +66,10 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", - **{f"{kind}-snapshots": f"{kind} snapshots" for kind in get_args(SnapshotType)}, + "snapshots": "snapshots", } -def snapshot_flag(kind: str) -> str: - """The flag a harness and a runtime both set for snapshots of `kind`.""" - return f"SUPPORTS_{kind.upper()}_SNAPSHOTS" - - # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -147,14 +142,12 @@ def supports(runtime: str, feature: str) -> bool: return isinstance(config, NetworkPolicyConfig) if feature == "services": return issubclass(runtime_cls(runtime), PrimeBackend) - if feature.endswith("-snapshots"): - flag = snapshot_flag(feature.removesuffix("-snapshots")) - return getattr(runtime_cls(runtime), flag, False) flag = { "container": "SUPPORTS_TASK_IMAGES", "user": "SUPPORTS_USER_SWITCHING", "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", + "snapshots": "SUPPORTS_FILESYSTEM_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -184,10 +177,8 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" - for need in needs: - kind = need.removesuffix("-snapshots") - if kind != need and not getattr(cls, snapshot_flag(kind), False): - return f"{name} cannot restore {kind} snapshots" + if "snapshots" in needs and not cls.SUPPORTS_FILESYSTEM_SNAPSHOTS: + return f"{name} cannot restore a snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index c9fce04d80..e2960a56bb 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -2,15 +2,11 @@ dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns before it, and finishes in a new harness process). -Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime -that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. -A new snapshot type adds its cases without a change here. The retention policies run +Every harness that declares `SUPPORTS_FILESYSTEM_SNAPSHOTS` runs it on prime, the one +runtime with snapshots; the others are collected as skips. The retention policies run on one harness: what they keep or delete does not depend on it. Every case checks that -the run leaks no snapshot: each one its boxes took is deleted on the platform, except -the ones the policy keeps.""" - -from collections.abc import Sequence -from typing import get_args +the run leaks no snapshot: each one its boxes took is deleted on the platform once the +run is final.""" import pytest from checkpoint_restore_v1 import FAIL_AFTER_TURNS @@ -18,36 +14,16 @@ from prime_sandboxes import AsyncSandboxClient from skill_codeword import SKILL -from verifiers.v1.configs.runtime import SnapshotType from verifiers.v1.harness import RESTORE_PROMPT -from verifiers.v1.rollout import kept from verifiers.v1.runtimes import PrimeBackend from verifiers.v1.types import content_text RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} +NEEDS = ("snapshots", "mcp") -def snapshot_grid( - harnesses: Sequence[str] | None = None, needs: tuple[str, ...] = ("mcp",) -) -> list: - """`(snapshot_type, harness, harness_runtime)` cells: the harness grid once per - snapshot type, each cell needing that type's snapshots.""" - return [ - pytest.param(kind, *cell.values, marks=cell.marks, id=f"{kind}-{cell.id}") - for kind in get_args(SnapshotType) - for cell in grid( - *([harnesses] if harnesses else []), needs=(f"{kind}-snapshots", *needs) - ) - ] - - -def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: - return { - "agent": { - "retries": RETRIES, - "checkpoint": {"type": snapshot_type, **checkpoint}, - } - } +def checkpoint_env(**checkpoint) -> dict: + return {"agent": {"retries": RETRIES, "checkpoint": checkpoint}} @pytest.fixture @@ -75,80 +51,86 @@ async def platform_state(snapshot_id: str) -> str | None: return None -async def assert_no_leaks(taken: list[str], kept_ids: set[str] = frozenset()) -> None: - """Every snapshot the run took is deleted, except `kept_ids`, which stay - restorable.""" - assert taken +async def assert_no_leaks(taken: list[str]) -> None: + """Every snapshot the run took is deleted.""" states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} leaked = { snapshot_id: state for snapshot_id, state in states.items() - if snapshot_id not in kept_ids and state not in (None, "DELETING") + if state not in (None, "DELETING") } assert leaked == {} - assert { - snapshot_id: states[snapshot_id] for snapshot_id in kept_ids - } == dict.fromkeys(kept_ids, "DURABLE") -@pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(), - indirect=["harness", "harness_runtime"], -) -async def test_retry_restores_checkpoint( - run, taken, snapshot_type, harness, harness_runtime -): +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): """The box restores to the state before the turn that failed, the turns up to there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not rerun, and the harness is told about the restore — the restored attempt builds on the earlier tool call's file. A snapshot every turn, so the failure at turn 3 restores a snapshot with turns behind it.""" - env = checkpoint_env(snapshot_type, interval=1) - trace = await restored_trace(run, harness, harness_runtime, env) + trace = await restored_trace(run, harness, harness_runtime, interval=1) assert trace.num_turns >= FAIL_AFTER_TURNS await assert_no_leaks(taken) @pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["bash"]), - indirect=["harness", "harness_runtime"], + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True ) -async def test_default_config_restores( - run, taken, snapshot_type, harness, harness_runtime -): - """With the default interval (every 10th turn), the retry restores the turn-1 - snapshot: the box as `setup` left it, before any tool call.""" - env = checkpoint_env(snapshot_type) - trace = await restored_trace(run, harness, harness_runtime, env) - assert len(trace.checkpoints) >= 1 - await assert_no_leaks(taken) +async def test_a_short_rollout_takes_no_snapshot(run, taken, harness, harness_runtime): + """With the default interval (every 10th turn), a rollout that fails at turn 3 + has taken no snapshot: its retry starts over and still succeeds.""" + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + assert trace.attempt == 2 + assert not trace.has_restored + assert trace.checkpoints == [] + assert taken == [] @pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["rlm"], needs=("mcp", "skills")), - indirect=["harness", "harness_runtime"], + "harness,harness_runtime", + grid(["rlm"], needs=(*NEEDS, "skills")), + indirect=True, ) -async def test_rlm_with_skills_restores( - run, taken, snapshot_type, harness, harness_runtime -): +async def test_rlm_with_skills_restores(run, taken, harness, harness_runtime): """rlm keeps its session in a per-run skill home when it has skills: the retry must load the session from the same home the restored box holds.""" config = harness.model_copy(update={"skills": [SKILL]}) - env = checkpoint_env(snapshot_type, interval=1) - trace = await restored_trace(run, config, harness_runtime, env) + trace = await restored_trace(run, config, harness_runtime, interval=1) assert trace.num_turns >= FAIL_AFTER_TURNS await assert_no_leaks(taken) -async def restored_trace(run, harness, harness_runtime, env): +@pytest.mark.parametrize("keep", ["last", 2], ids=["keep-last", "keep-every-2nd"]) +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_checkpoint_retention(run, taken, harness, harness_runtime, keep): + """Snapshots the policy drops are deleted while the rollout runs, without breaking + the newest one: the retry still restores it. Every snapshot is deleted once the + run is final.""" + trace = await restored_trace(run, harness, harness_runtime, interval=1, keep=keep) + assert all(checkpoint.deleted for checkpoint in trace.checkpoints) + await assert_no_leaks(taken) + + +async def restored_trace(run, harness, harness_runtime, **checkpoint): (episode,) = await run( "checkpoint-restore-v1", harness=harness, runtime=harness_runtime, - env=env, + env=checkpoint_env(**checkpoint), max_turns=12, max_tokens=4096, ) @@ -163,6 +145,8 @@ async def restored_trace(run, harness, harness_runtime, env): assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] assert trace.has_restored assert trace.attempt == 2 + # Each checkpoint records what its snapshot captured. + assert {c.snapshot.type for c in trace.checkpoints} == {"filesystem"} # The error belongs to the first attempt, in the agent stage. assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] # The restored process made no side request of its own (it would start a second @@ -181,49 +165,3 @@ async def restored_trace(run, harness, harness_runtime, env): ] assert any(call.node is None for call in trace.calls) return trace - - -@pytest.mark.parametrize( - "keep,after", - [("last", "delete"), ("last", "keep"), ("all", "keep"), (2, "keep")], - ids=["keep-last-delete", "keep-last", "keep-all", "keep-every-2nd"], -) -@pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["bash"]), - indirect=["harness", "harness_runtime"], -) -async def test_checkpoint_retention( - run, taken, snapshot_type, harness, harness_runtime, keep, after -): - """Snapshots the policy drops are deleted while the rollout runs, without breaking - the newest one: the retry still restores it. When the run is final, `after` deletes - the rest or leaves exactly what `keep` holds.""" - (episode,) = await run( - "checkpoint-restore-v1", - harness=harness, - runtime=harness_runtime, - env=checkpoint_env(snapshot_type, interval=1, keep=keep, after=after), - max_turns=12, - max_tokens=4096, - ) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - assert trace.reward == 1.0 - assert trace.has_restored - live = [c.id for c in trace.checkpoints if not c.deleted] - try: - if after == "delete": - assert live == [] - else: - assert set(live) == kept(trace.checkpoints, keep) - if keep == "last": - assert len(live) == 1 - if keep == "all": - # Snapshots still uploading when the run succeeded were never restorable. - assert live == [c.id for c in trace.checkpoints if c.time.end] - await assert_no_leaks(taken, set(live)) - finally: - async with AsyncSandboxClient() as client: - for checkpoint in live: - await client.delete_checkpoint(checkpoint) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 004a086e22..ebf2398fd4 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -156,7 +156,7 @@ async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: def make_rollout( - interval: int = 1, keep: Literal["all", "last"] | int = "last" + interval: int = 1, keep: Literal["last"] | int = "last" ) -> tuple[Rollout, SlowBackend]: backend = SlowBackend() run = Rollout.__new__(Rollout) @@ -175,6 +175,7 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] + run._restored_turn = None return run, backend @@ -183,15 +184,22 @@ async def checkpoint_state() -> dict: async def test_checkpoint_interval_skips_turns(): run, backend = make_rollout(interval=2) - assert await run._checkpoint(TURN) is None # turn 2: 1 % 2 + assert await run._checkpoint(TURN) is not None # turn 2: the 2nd run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) run.trace.nodes.append( MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) ) - assert await run._checkpoint(TURN) is not None # turn 3: 2 % 2 + assert await run._checkpoint(TURN) is None # turn 3 assert backend.taken == 1 +async def test_the_restored_turn_is_not_snapshotted_again(): + run, backend = make_rollout(interval=2) + run._restored_turn = run.trace.num_turns + 1 + assert await run._checkpoint(TURN) is None + assert backend.taken == 0 + + async def test_checkpoint_becomes_durable_in_the_background(): run, backend = make_rollout(interval=1) checkpoint = await run._checkpoint(TURN) @@ -244,7 +252,7 @@ def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: def test_keep_policies(): checkpoints = durable("a", "b", "c", "d", "e") assert kept(checkpoints, "last") == {"e"} - assert kept(checkpoints, "all") == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 1) == {"a", "b", "c", "d", "e"} assert kept(checkpoints, 2) == {"a", "c", "e"} assert kept(checkpoints, 3) == {"a", "d", "e"} # Deleted and not yet durable snapshots are never kept. @@ -252,7 +260,7 @@ def test_keep_policies(): *durable("a", "b", deleted=("a",)), Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), ] - assert kept(pending, "all") == {"b"} + assert kept(pending, 1) == {"b"} async def test_a_durable_snapshot_prunes_the_older_ones(): diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index f71349fef3..b4d9844c0a 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -96,7 +96,6 @@ RuntimeInfo, RuntimeProcess, Snapshot, - SnapshotType, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -282,7 +281,6 @@ "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", "Snapshot": "verifiers.v1.runtimes", - "SnapshotType": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplingMask": "verifiers.v1.types", @@ -494,7 +492,6 @@ "RuntimeConfig", "RuntimeInfo", "Snapshot", - "SnapshotType", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d46400368c..d89e1725d4 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -35,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.rollout import Rollout, RolloutTimeouts, kept +from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -819,19 +819,12 @@ async def run( await asyncio.sleep(delay) attempt += 1 finally: - # The run is final (or cancelled): delete what `after` does not keep. Every - # box is gone by now, so the platform refuses none of them. - keep = ( - kept(run.trace.checkpoints, checkpointing.keep) - if run is not None - and checkpointing is not None - and checkpointing.after == "keep" - else set() - ) + # The run is final (or cancelled): delete every snapshot left. Every box + # is gone by now, so the platform refuses none of them. drop = [ snapshot for snapshot_id, snapshot in sorted(snapshots.items()) - if snapshot_id not in deleted and snapshot_id not in keep + if snapshot_id not in deleted ] if drop and run is not None and run.runtime is not None: try: diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 090456a3dc..384a4c96c8 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -17,7 +17,7 @@ from verifiers.v1.configs.deployment import DeploymentConfig from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig, SnapshotType +from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig from verifiers.v1.configs.sampling import SamplingConfig from verifiers.v1.configs.scoring import ScoringConfig from verifiers.v1.utils.generic import merge_defaults @@ -40,23 +40,19 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing: snapshot the box at model turns, so a retry continues - from the newest snapshot instead of starting over. Needs a runtime and a harness - that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts over. Processes - the agent left running (background jobs, a REPL) are gone after a restore.""" + """Rollout checkpointing: snapshot the box every n-th model turn, so a retry + continues from the newest snapshot instead of starting over. The runtime takes the + best snapshot it supports, and each checkpoint records its type. Needs a runtime + and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts + over. Every snapshot is deleted once the run is final.""" - type: SnapshotType = "filesystem" - """What a snapshot captures.""" interval: PositiveInt = 10 - """Snapshot at every n-th model turn (the 1st, the 1+n-th, ...); a restore loses - up to n turns.""" - keep: Literal["all", "last"] | PositiveInt = "last" - """Snapshots kept while the rollout runs: the newest (`"last"`), every one - (`"all"`), or every k-th plus the newest (`k`). A failed restore falls back to an - older kept one.""" - after: Literal["delete", "keep"] = "delete" - """What happens to the kept snapshots once the run is final: `"delete"` them, or - `"keep"` them for their owner to delete.""" + """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter + than n turns takes none, and a restore loses fewer than n turns.""" + keep: Literal["last"] | PositiveInt = "last" + """Snapshots kept while the rollout runs: the newest (`"last"`), or every k-th plus + the newest (`k`; 1 keeps every one). A failed restore falls back to an older kept + one.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 716dadb1ce..2ac1fcdc4e 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -14,10 +14,6 @@ parse_network_rule, ) -SnapshotType = Literal["filesystem"] -"""What a runtime snapshot captures: `filesystem` holds the box's files, not its -processes.""" - class BindMount(BaseConfig): """An existing file or directory on the container engine's host.""" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 76213f9a57..00ef74e7be 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -113,10 +113,10 @@ async def resolve_agent_runtime( return view -def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> set[str]: +def kept(checkpoints: list[Checkpoint], keep: Literal["last"] | int) -> set[str]: """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` - policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every one for - `"all"` or every k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every + k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" live = [ (index, c) for index, c in enumerate(checkpoints) @@ -124,8 +124,6 @@ def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> s ] if not live: return set() - if keep == "all": - return {c.id for _, c in live} held = {live[-1][1].id} if keep != "last": held.update(c.id for index, c in live if index % keep == 0) @@ -164,8 +162,7 @@ def __init__( model turns go upstream (on a runtime and a harness that support it) for a retry to restore, and deletes the ones its `keep` policy drops as newer ones become restorable; the owner of the retries deletes the rest once the run is - final (`RolloutCheckpointConfig.after`) — this rollout deletes them only when it - aborts.""" + final — this rollout deletes them only when it aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -192,6 +189,9 @@ def __init__( """The snapshots still on their way to restorable, by checkpoint id.""" self._attempt = attempt self._restore: Checkpoint | None = None + self._restored_turn: int | None = None + """The turn the restored snapshot was taken at, which this run does not + snapshot again.""" self._harness_data: TaskData | None = None self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" @@ -203,6 +203,7 @@ def __init__( self.trace, self._restore = restore self.trace.attempt = attempt self.rewound_snapshots = self.trace.rewind(self._restore) + self._restored_turn = self.trace.num_turns + 1 self._restore.restored = True else: self.trace = Trace( @@ -826,7 +827,11 @@ async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: assert self.runtime is not None assert self._checkpointing is not None - if self.trace.num_turns % self._checkpointing.interval: + turn_number = self.trace.num_turns + 1 + if ( + turn_number % self._checkpointing.interval + or turn_number == self._restored_turn + ): return None if len(self.trace._pending) > 1: # Another request is in flight (parallel sub-agents share the trace): the @@ -986,18 +991,16 @@ async def abort(self) -> None: with contextlib.suppress(Exception): await self.runtime.backend.stop() if self.runtime is not None: - # Snapshots kept `after` the run belong to their owner, an aborted run's too; - # strays were never restorable, so nobody else knows of them. - owned = self._checkpointing is None or self._checkpointing.after == "delete" + # Strays were never restorable, so nobody else knows of them. with contextlib.suppress(Exception): self.record_deleted( await self.runtime.backend.discard_snapshots( [ - *(self.rewound_snapshots if owned else ()), + *self.rewound_snapshots, *( c.snapshot for c in self.trace.checkpoints - if owned and not c.deleted + if not c.deleted ), *self.runtime.backend.stray_snapshots, ] diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 7081299e8b..1acfc71a09 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -29,7 +29,6 @@ RuntimeBackend, RuntimeProcess, Snapshot, - SnapshotType, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -111,7 +110,6 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "RuntimeInfo", "RuntimeProcess", "Snapshot", - "SnapshotType", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 722df77fd4..71c354fba6 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,12 +13,12 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal from pydantic import BaseModel from pydantic_config import BaseConfig -from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType +from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import SandboxError from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.format import format_time @@ -149,7 +149,9 @@ class Snapshot(BaseModel): id: str """The provider's id for the snapshot.""" - type: SnapshotType = "filesystem" + type: Literal["filesystem"] = "filesystem" + """What the snapshot captures, as the runtime took it: `filesystem` holds the box's + files, not its processes.""" @dataclass diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 8d374b1391..0ae847959c 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -418,8 +418,8 @@ class Checkpoint(BaseModel): restored: bool = False """Whether a retry restored the run from this snapshot.""" deleted: bool = False - """Whether the snapshot was deleted (`RolloutCheckpointConfig.keep`, `.after`): it - can no longer be restored.""" + """Whether the snapshot was deleted (by `RolloutCheckpointConfig.keep`, or once the + run is final): it can no longer be restored.""" def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: From 8b20e184c37f9cb179cddf95fbcbeb5e24b914c8 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 00:13:19 +0000 Subject: [PATCH 20/53] derive restorable harnesses and step back after a dead restore A harness can continue a rollout in a box restored from a snapshot when a new process of it can take the rollout over. For a harness that resumes on the transcript (SUPPORTS_RESUME), a restore is that same relaunch, so it needs nothing more. An ACP harness keeps its session in a live process and needs its agent to load the session from its files: ACPHarness.LOADS_SESSIONS (claude-code, codex, rlm). This replaces the harness-side SUPPORTS_FILESYSTEM_SNAPSHOTS (Harness.can_restore). browser-use now restores like any harness that resumes on its transcript. A restore whose attempt fails before its first new turn is marked failed like one whose box never booted, so the next retry steps back to an older snapshot or starts over instead of restoring the same snapshot until the retries run out. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 2 +- docs/harnesses.md | 2 +- tests/integration/grid.py | 6 +-- tests/integration/test_checkpoint.py | 4 +- tests/unit/test_checkpoint.py | 4 +- tests/unit/test_retry_ids.py | 53 +++++++++++++++++++ verifiers/v1/acp/__init__.py | 11 +++- verifiers/v1/agent.py | 22 +++++--- verifiers/v1/configs/agent.py | 3 +- verifiers/v1/harness.py | 15 +++--- verifiers/v1/harnesses/bash/harness.py | 1 - verifiers/v1/harnesses/claude_code/harness.py | 2 +- verifiers/v1/harnesses/codex/harness.py | 2 +- verifiers/v1/harnesses/null/harness.py | 1 - verifiers/v1/harnesses/rlm/harness.py | 2 +- verifiers/v1/rollout.py | 14 ++--- 16 files changed, 106 insertions(+), 38 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 6c4f4cf0ca..8aabc1e8aa 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,7 +77,7 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). Elsewhere a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] diff --git a/docs/harnesses.md b/docs/harnesses.md index 624e909037..67ad0f0503 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that a new process of the harness can continue a rollout in a box restored from a filesystem snapshot (`agent.checkpoint`). Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. +- `LOADS_SESSIONS` (ACP harnesses): indicates that the pinned agent implements ACP `session/load`, so a new process can continue a rollout in a box restored from a snapshot (`agent.checkpoint`). A harness with `SUPPORTS_RESUME` needs nothing more: a restore relaunches it on the transcript. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 12e8a07a91..1b163aa4bf 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,8 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). A `snapshots` need asks both the harness and -the runtime for `SUPPORTS_FILESYSTEM_SNAPSHOTS`. +`SUPPORTS_RESUME` or a native session). A `snapshots` need asks the runtime for +`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -177,7 +177,7 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" - if "snapshots" in needs and not cls.SUPPORTS_FILESYSTEM_SNAPSHOTS: + if "snapshots" in needs and not cls.can_restore(): return f"{name} cannot restore a snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index e2960a56bb..e8e872159c 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -2,8 +2,8 @@ dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns before it, and finishes in a new harness process). -Every harness that declares `SUPPORTS_FILESYSTEM_SNAPSHOTS` runs it on prime, the one -runtime with snapshots; the others are collected as skips. The retention policies run +Every harness that can continue in a new process (`Harness.can_restore`) runs it on +prime, the one runtime with snapshots; the others are collected as skips. The retention policies run on one harness: what they keep or delete does not depend on it. Every case checks that the run leaks no snapshot: each one its boxes took is deleted on the platform once the run is final.""" diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index ebf2398fd4..06ede65093 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -175,7 +175,7 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] - run._restored_turn = None + run.restored_turns = None return run, backend @@ -195,7 +195,7 @@ async def test_checkpoint_interval_skips_turns(): async def test_the_restored_turn_is_not_snapshotted_again(): run, backend = make_rollout(interval=2) - run._restored_turn = run.trace.num_turns + 1 + run.restored_turns = run.trace.num_turns assert await run._checkpoint(TURN) is None assert backend.taken == 0 diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 45c540ecf8..3e46decd6f 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -6,6 +6,7 @@ import verifiers.v1 as vf import verifiers.v1.agent as agent_module import verifiers.v1.utils.retries as retries_module +from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.env import Env, RunSlot RETRIES = vf.RetryConfig(max_retries=2) @@ -65,6 +66,58 @@ async def close(self) -> vf.Trace: assert [e.type for e in trace.errors] == ["SandboxError", "SandboxError"] +@pytest.mark.asyncio +async def test_a_restore_that_leads_nowhere_is_not_picked_again(monkeypatch): + """Attempt 1 takes a snapshot and fails. Attempt 2 restores it and fails before its + first new turn, as a harness that cannot load its session would every time. The + next retry starts over instead of restoring the same snapshot again.""" + restores: list[str | None] = [] + + class FakeRollout: + runtime = None + + def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): + restores.append(None if restore is None else restore[1].id) + self.rewound_snapshots: list[vf.Snapshot] = [] + self.deleted_snapshots: set[str] = set() + self.restored_turns = None + if restore is None: + self.trace = make_trace(id=trace_id, attempt=attempt) + else: + self.trace = restore[0] + self.trace.attempt = attempt + self.restored_turns = self.trace.num_turns + on_trace(self.trace) + + async def open(self) -> bool: + if self.trace.attempt == 1: + self.trace.checkpoints.append( + vf.Checkpoint( + id="c1", + snapshot=vf.Snapshot(id="c1"), + time=vf.TimeSpan(start=1.0, end=2.0), + ) + ) + if self.trace.attempt < 3: + self.trace.errors.append(FAILURE) + else: + self.trace.ok = True + return False + + async def close(self) -> vf.Trace: + return self.trace + + monkeypatch.setattr(agent_module, "Rollout", FakeRollout) + agent = vf.Agent(vf.AgentConfig(model="test-model", retries=RETRIES)) + monkeypatch.setattr(agent, "_rollout_params", lambda *args: {}) + monkeypatch.setattr(agent, "_checkpointing", lambda: RolloutCheckpointConfig()) + + trace = await agent.run(vf.Task(vf.TaskData(idx=0, prompt=None))) + + assert restores == [None, "c1", None] + assert trace.ok and trace.attempt == 3 + + @pytest.mark.asyncio async def test_episode_retries_keep_the_episode_id(): calls: list[tuple[str, int]] = [] diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index d5fec9315b..22169fbbf9 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -7,7 +7,7 @@ from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeAlias, TypeVar, cast +from typing import Any, ClassVar, TypeAlias, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field @@ -70,6 +70,15 @@ class ACPConfig: class ACPHarness(Harness[ConfigT]): """Harness backed by one live ACP process and native session per rollout.""" + LOADS_SESSIONS: ClassVar[bool] = False + """Whether the pinned agent implements ACP `session/load`, so a new process can + continue a session from its files in the box. An agent that does not advertise + `loadSession` at runtime takes no snapshots either way.""" + + @classmethod + def can_restore(cls) -> bool: + return cls.LOADS_SESSIONS + async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script( ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"} diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d89e1725d4..b3f3b10a2e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -663,17 +663,14 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if ( - harness.SUPPORTS_FILESYSTEM_SNAPSHOTS - and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - ): + if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: return config key = (harness.config.id, self.config.runtime.type) if key not in _checkpointing_warned: _checkpointing_warned.add(key) reason = ( f"harness {harness.config.id!r} does not support it" - if not harness.SUPPORTS_FILESYSTEM_SNAPSHOTS + if not harness.can_restore() else f"the {self.config.runtime.type!r} runtime takes no snapshots" ) logger.warning( @@ -774,10 +771,19 @@ async def run( (c.snapshot.id, c.snapshot) for c in trace.checkpoints ) deleted.update(run.deleted_snapshots) - if target is not None and any( - error.stage == "setup" for error in trace.errors[seen:] + if ( + target is not None + and any(error.stage == "setup" for error in trace.errors[seen:]) + or ( + not trace.ok + and run.restored_turns is not None + and trace.num_turns <= run.restored_turns + ) ): - # The box never booted from it: the next retry steps back. + # The restore led nowhere: the box never booted from it, or the + # attempt failed before its first new turn (a harness that + # cannot load its session fails the same way every time). The + # next retry steps back to an older snapshot, or starts over. failed_restores.add(target.id) if trace.ok: break diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 384a4c96c8..7a920b279c 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -43,7 +43,8 @@ class RolloutCheckpointConfig(BaseConfig): """Rollout checkpointing: snapshot the box every n-th model turn, so a retry continues from the newest snapshot instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. Needs a runtime - and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts + that takes snapshots (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new + process can continue the rollout (`Harness.can_restore`); elsewhere a retry starts over. Every snapshot is deleted once the run is final.""" interval: PositiveInt = 10 diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index b91ff74957..84b7c10f46 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -63,13 +63,6 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" - SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False - """Whether a new process of this harness can continue a rollout in a box restored - from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the - newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout - checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_SNAPSHOTS` too. Distinct - from `SUPPORTS_RESUME`: a restore never finds a live process, its program - continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True """Whether the program hands the model local execution in the runtime — true for every real harness; the tool-less chat loops (`null`) override to False. Read @@ -85,6 +78,14 @@ class Harness(ABC, Generic[ConfigT]): leaks host state (auth, config, processes) both ways. Only the minimal in-house loops (`bash`, `null`) override to False.""" + @classmethod + def can_restore(cls) -> bool: + """Whether a new process of this harness can continue a rollout in a box + restored from a snapshot (`AgentConfig.checkpoint`): the default relaunches on + the trace's transcript, so a harness that resumes that way (`SUPPORTS_RESUME`) + can.""" + return cls.SUPPORTS_RESUME + def __init__(self, config: ConfigT) -> None: self.config = config diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 3a7f2e8b70..8f6d852144 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,7 +52,6 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index d66d9904b0..f4b7462916 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -42,7 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index fb50cc6481..3b28c64fcd 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -63,7 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 252a23118f..503429ddb6 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,7 +26,6 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 8aac1ca420..61f49228ea 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -142,7 +142,7 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 00ef74e7be..9d507ef30e 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -189,9 +189,10 @@ def __init__( """The snapshots still on their way to restorable, by checkpoint id.""" self._attempt = attempt self._restore: Checkpoint | None = None - self._restored_turn: int | None = None - """The turn the restored snapshot was taken at, which this run does not - snapshot again.""" + self.restored_turns: int | None = None + """The model turns the restored trace held when this run began (None for a run + that did not restore). The next turn is the one the snapshot was taken at, which + this run does not snapshot again.""" self._harness_data: TaskData | None = None self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" @@ -203,7 +204,7 @@ def __init__( self.trace, self._restore = restore self.trace.attempt = attempt self.rewound_snapshots = self.trace.rewind(self._restore) - self._restored_turn = self.trace.num_turns + 1 + self.restored_turns = self.trace.num_turns self._restore.restored = True else: self.trace = Trace( @@ -828,9 +829,8 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: assert self.runtime is not None assert self._checkpointing is not None turn_number = self.trace.num_turns + 1 - if ( - turn_number % self._checkpointing.interval - or turn_number == self._restored_turn + if turn_number % self._checkpointing.interval or ( + self.restored_turns is not None and turn_number == self.restored_turns + 1 ): return None if len(self.trace._pending) > 1: From 014d6389890559ee012c64767eae05ced8a9c186 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 00:19:57 +0000 Subject: [PATCH 21/53] fix the dead-restore check's grouping The condition read `target is not None and A or B`, so B ran for runs that restored nothing and read `restored_turns` on them. Group it as `target is not None and (A or B)`. The new test's fake rollout also called the run's `on_trace` hook, which it does not pass. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_retry_ids.py | 1 - verifiers/v1/agent.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 3e46decd6f..3a046037bc 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -87,7 +87,6 @@ def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): self.trace = restore[0] self.trace.attempt = attempt self.restored_turns = self.trace.num_turns - on_trace(self.trace) async def open(self) -> bool: if self.trace.attempt == 1: diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index b3f3b10a2e..df4be8c0c9 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -771,9 +771,8 @@ async def run( (c.snapshot.id, c.snapshot) for c in trace.checkpoints ) deleted.update(run.deleted_snapshots) - if ( - target is not None - and any(error.stage == "setup" for error in trace.errors[seen:]) + if target is not None and ( + any(error.stage == "setup" for error in trace.errors[seen:]) or ( not trace.ok and run.restored_turns is not None From 10e467024863ab171e2f4258c3e975d8872a108c Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:19:52 +0000 Subject: [PATCH 22/53] restore rollout retries from memory snapshots A memory snapshot (a pause of the box) holds the box's processes. A retry boots a new box from it, reroutes the box's relay to the new session and waits for the harness program the box still runs, so no harness has to load a session or be told about the restore. The ACP runner can run as a job of the box with one job per request, because no connection from the host survives a stopped box. Co-Authored-By: Claude --- .../integration/fixtures/memory_restore_v1.py | 98 ++++++++++ tests/integration/grid.py | 7 +- tests/integration/test_memory_checkpoint.py | 134 +++++++++++++ verifiers/v1/acp/__init__.py | 141 +++++++++++++- verifiers/v1/acp/runner.py | 136 ++++++++++--- verifiers/v1/agent.py | 6 + verifiers/v1/configs/agent.py | 19 +- verifiers/v1/configs/runtime.py | 3 + verifiers/v1/harness.py | 43 +++- verifiers/v1/harnesses/bash/harness.py | 1 + verifiers/v1/harnesses/null/harness.py | 1 + verifiers/v1/harnesses/utils/launch.py | 5 +- verifiers/v1/interception/relay/__init__.py | 20 +- verifiers/v1/interception/server.py | 6 + verifiers/v1/rollout.py | 183 ++++++++++++++++-- verifiers/v1/runtimes/__init__.py | 2 + verifiers/v1/runtimes/backends/prime.py | 122 +++++++++++- verifiers/v1/runtimes/base.py | 40 +++- verifiers/v1/runtimes/runtime.py | 18 +- verifiers/v1/session.py | 15 ++ verifiers/v1/trace.py | 8 + 21 files changed, 938 insertions(+), 70 deletions(-) create mode 100644 tests/integration/fixtures/memory_restore_v1.py create mode 100644 tests/integration/test_memory_checkpoint.py diff --git a/tests/integration/fixtures/memory_restore_v1.py b/tests/integration/fixtures/memory_restore_v1.py new file mode 100644 index 0000000000..3891c6bfd4 --- /dev/null +++ b/tests/integration/fixtures/memory_restore_v1.py @@ -0,0 +1,98 @@ +"""memory-restore: a task whose first attempt dies mid-rollout, so the retry restores the +box's newest memory snapshot and continues the harness program the box still runs. + +`setup` starts a process in the box that holds a nonce only in its memory and hands it +to whoever reads `nonce.fifo`. The nonce is on no disk, so a box that boots from a +filesystem snapshot cannot answer; a box that continues from a memory snapshot can. +The reward reads the nonce from the process. The task asks a harness with a shell for +two commands that build on each other, and a metric checks the file they leave. + +A response `@intercept` raises a `ProviderError` once, at the model turn `fail_turn`: +the harness gets the error as its reply and dies, after the snapshot of that turn +exists. The restored box still waits for that turn's reply, gets it from the retry, and +finishes: no new harness process, no notice to the model. +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +FIFO = "nonce.fifo" +STEP_ONE = "step1.txt" +DONE = "done.txt" +READ = f"timeout 10 cat {FIFO}" +# The nonce is the script's first argument: it lives in the process, not in a file. +SERVER = f'rm -f {FIFO}; mkfifo {FIFO}; while :; do printf %s "$1" > {FIFO}; done' + + +class MemoryRestoreTaskConfig(vf.TaskConfig): + fail_turn: int = 2 + """The model turn whose reply fails once (from 1).""" + + +class MemoryRestoreConfig(vf.TasksetConfig): + task: MemoryRestoreTaskConfig = MemoryRestoreTaskConfig() + + +class MemoryRestoreTask(vf.Task[vf.TaskData, vf.State, MemoryRestoreTaskConfig]): + failed = False + """Whether the injected failure fired: once per task, so a retry from the start + (checkpoints off) also gets through.""" + + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.run_background( + ["sh", "-c", SERVER, "sh", nonce], {}, "server.log" + ) + + @vf.intercept + def fail_once(self, response: vf.Response, trace: vf.Trace) -> None: + if trace.num_turns + 1 >= self.config.fail_turn and not self.failed: + self.failed = True + # A 4xx: harness SDKs end the turn at once instead of retrying it until the + # rollout's deadline. + raise vf.ProviderError("injected mid-rollout failure", status_code=400) + + async def _read(self, runtime: Runtime, command: str) -> str: + try: + result = await runtime.run(["sh", "-c", command], {}) + except (SandboxError, OSError): + return "" + return result.stdout.strip() + + @vf.reward(weight=1.0) + async def process_alive(self, trace: vf.Trace, runtime: Runtime) -> float: + """The process `setup` started still runs and still holds the nonce.""" + return float(await self._read(runtime, READ) == trace.info["nonce"]) + + @vf.metric + async def copied(self, trace: vf.Trace, runtime: Runtime) -> float: + """The agent's two commands left the nonce in the done file.""" + return float(await self._read(runtime, f"cat {DONE}") == trace.info["nonce"]) + + +class MemoryRestoreTaskset(vf.Taskset[MemoryRestoreTask, MemoryRestoreConfig]): + def load(self) -> list[MemoryRestoreTask]: + return [ + MemoryRestoreTask( + vf.TaskData( + idx=0, + prompt=( + "Run these shell commands in the working directory, one " + "command per turn, and wait for each result before the " + "next. If you cannot run commands, reply with the single " + "word: done.\n" + f"1. `{READ} > {STEP_ONE}`\n" + f"2. `cp {STEP_ONE} {DONE}`\n" + "3. Reply with the single word: done." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["MemoryRestoreTaskset"] diff --git a/tests/integration/grid.py b/tests/integration/grid.py index a4b2c39654..5ec8c84a0e 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -12,7 +12,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via `SUPPORTS_RESUME` or a native session). A `snapshots` need asks the runtime for -`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`. +`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`; a `memory` +need asks them for `SUPPORTS_MEMORY_SNAPSHOTS` and `Harness.SUPPORTS_MEMORY_RESTORE`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -68,6 +69,7 @@ "compose": "Compose services", "services": "native service VMs", "snapshots": "snapshots", + "memory": "memory snapshots", } @@ -152,6 +154,7 @@ def supports(runtime: str, feature: str) -> bool: "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", "snapshots": "SUPPORTS_FILESYSTEM_SNAPSHOTS", + "memory": "SUPPORTS_MEMORY_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -183,6 +186,8 @@ def unsupported( return f"{name} cannot resume an exchange" if "snapshots" in needs and not cls.can_restore(): return f"{name} cannot restore a snapshot" + if "memory" in needs and not cls.SUPPORTS_MEMORY_RESTORE: + return f"{name} cannot continue from a memory snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py new file mode 100644 index 0000000000..800ba5b6a5 --- /dev/null +++ b/tests/integration/test_memory_checkpoint.py @@ -0,0 +1,134 @@ +"""A retry restores the box's newest memory snapshot (memory-restore-v1: the reply of one +model turn fails once; the retry boots a new box from the memory snapshot of that turn, +and the harness program the box still runs gets the reply and finishes). + +Every harness that continues from a memory snapshot (`Harness.SUPPORTS_MEMORY_RESTORE`) +runs it on prime, the one runtime with memory snapshots. A process the task's `setup` +started holds a nonce only in its memory: the reward reads it from the restored box, so +a box that booted from the snapshot's files alone would fail. Every case checks that the +run leaks no snapshot.""" + +import pytest +from grid import grid +from prime_sandboxes import AsyncSandboxClient + +from verifiers.v1.harness import RESTORE_PROMPT +from verifiers.v1.runtimes import PrimeBackend +from verifiers.v1.types import content_text +from verifiers.v1.utils.loaders import harness_class + +RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} +NEEDS = ("memory",) +SHELLS = ["bash", "rlm"] + + +@pytest.fixture +def taken(monkeypatch) -> list[tuple[str, str]]: + """The id and type of every snapshot the run's boxes take, in every attempt.""" + snapshots: list[tuple[str, str]] = [] + snapshot = PrimeBackend.snapshot + + async def recording(self, **kwargs): + result = await snapshot(self, **kwargs) + snapshots.append((result.id, result.type)) + return result + + monkeypatch.setattr(PrimeBackend, "snapshot", recording) + return snapshots + + +async def assert_no_leaks(taken: list[tuple[str, str]]) -> None: + """Every snapshot the run took is deleted on the platform.""" + leaked = {} + async with AsyncSandboxClient() as client: + for snapshot_id, _ in taken: + try: + state = (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + continue + if state != "DELETING": + leaked[snapshot_id] = state + assert leaked == {} + + +async def restored_trace( + run, harness, harness_runtime, *, fail_turn: int, **checkpoint +): + (episode,) = await run( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": fail_turn}}, + env={"agent": {"retries": RETRIES, "checkpoint": checkpoint}}, + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + # The process `setup` started kept its memory across the restore. + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] + assert trace.attempt == 2 + assert trace.has_restored + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + assert {c.snapshot.type for c in trace.checkpoints} == {"memory"} + # The harness program never restarted: one root, and no restore notice. + assert sum(node.parent is None for node in trace.nodes) == 1 + assert not any( + node.message.role == "user" + and RESTORE_PROMPT in content_text(node.message.content) + for node in trace.nodes + ) + # Every turn kept has its call; the failed turn's call stays, unlinked. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace + + +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_memory_checkpoint(run, taken, harness, harness_runtime): + """A snapshot at every turn. A harness with a shell fails at its second turn, so the + restored box holds the first turn's command and its harness goes on to the second; + a harness without one has a single turn, which fails.""" + shell = harness_class(harness.id).EXECUTES_CODE + trace = await restored_trace( + run, harness, harness_runtime, fail_turn=2 if shell else 1, interval=1 + ) + if shell: + assert trace.num_turns >= 2 + assert trace.metrics["copied"] == 1.0 + assert {kind for _, kind in taken} == {"memory"} + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_filesystem_snapshots_stay_selectable( + run, taken, harness, harness_runtime +): + """`snapshot = "filesystem"` takes no memory snapshot: the retry boots from the + snapshot's files, where the process that held the nonce is gone.""" + (episode,) = await run( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + env={ + "agent": { + "retries": RETRIES, + "checkpoint": {"interval": 1, "snapshot": "filesystem"}, + } + }, + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.has_restored + assert {kind for _, kind in taken} == {"filesystem"} + assert trace.reward == 0.0 + await assert_no_leaks(taken) diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 22169fbbf9..bd1b66a992 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -3,6 +3,9 @@ import asyncio import contextlib import json +import shlex +import time +import uuid from abc import abstractmethod from collections.abc import AsyncIterator from dataclasses import dataclass @@ -16,6 +19,7 @@ from verifiers.v1.errors import HarnessError, InterceptionError from verifiers.v1.harness import RESTORE_PROMPT, Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess +from verifiers.v1.runtimes.base import current_program from verifiers.v1.semantic import ( ACP_SEMANTIC_EDGES_METADATA_KEY, SemanticEdgeSet, @@ -27,6 +31,9 @@ ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() MAX_PACKET_BYTES = 128 * 1024 * 1024 +ACP_ROOT = "/tmp/vf-acp" +"""Where a detached runner keeps its socket, log and request files, per rollout.""" +RUNNER_START_SECONDS = 120 _STOP_CONDITIONS = { "max_tokens": "max_output_tokens", "max_turn_requests": "max_turns", @@ -75,6 +82,8 @@ class ACPHarness(Harness[ConfigT]): continue a session from its files in the box. An agent that does not advertise `loadSession` at runtime takes no snapshots either way.""" + SUPPORTS_MEMORY_RESTORE: ClassVar[bool] = True + @classmethod def can_restore(cls) -> bool: return cls.LOADS_SESSIONS @@ -250,6 +259,19 @@ def __init__( """The agent's ACP session id, once a checkpoint asked for it.""" self._restore_id: str | None = None """The session the next process loads from the restored box (`restore`).""" + self._detached: list[str] | None = None + """The runner's argv once it runs as a job of the box (`memory`): each request + is then a job too, since no connection from the host survives a stopped box.""" + self._prompt: dict[str, Any] | None = None + """The reply file and the trace's call count of the prompt request under way.""" + + @property + def _dir(self) -> str: + return f"{ACP_ROOT}/{self.trace.id}" + + @property + def _running(self) -> bool: + return self._process is not None or self._detached is not None async def _start(self) -> None: self._stderr_tail.clear() @@ -258,6 +280,23 @@ async def _start(self) -> None: {**self.config.env, "UV_FROZEN": "false"}, activate=False, ) + if self.memory: + socket = f"{self._dir}/socket" + await self.runtime.run(["mkdir", "-p", self._dir], {}) + await self.runtime.run_background( + [*program, "--socket", socket], + self.config.env, + log=f"{self._dir}/runner.log", + ) + deadline = time.monotonic() + RUNNER_START_SECONDS + while (await self.runtime.run(["test", "-S", socket], {})).exit_code: + if time.monotonic() > deadline: + raise RuntimeError( + f"ACP runner did not start: {await self._runner_log()}" + ) + await asyncio.sleep(0.2) + self._detached = program + return process = await self.runtime.exec(program, self.config.env) self._process = process self._reader = _PacketReader(process.stdout) @@ -278,7 +317,71 @@ async def _read_responses(self, reader: _PacketReader) -> None: RuntimeError(f"ACP connection closed: {error}") ) + async def _runner_log(self) -> str: + with contextlib.suppress(Exception): + log = await self.runtime.run( + ["tail", "-c", "4000", f"{self._dir}/runner.log"], {} + ) + return log.stdout.strip() + return "" + + async def _call(self, operation: str, payload: dict[str, Any]) -> JsonObject: + """One request to the detached runner, as a job of the box.""" + assert self._detached is not None + name = uuid.uuid4().hex + request, reply = f"{self._dir}/{name}.request", f"{self._dir}/{name}.reply" + await self.runtime.write( + request, + json.dumps({"id": name, "operation": operation, **payload}).encode(), + ) + program = None + if operation == "prompt": + program = current_program.get() + self._prompt = {"reply": reply, "calls_before": len(self.trace.calls)} + try: + result = await self.runtime.run( + [*self._detached, "--call", f"{self._dir}/socket", request, reply], + self.config.env, + program=program, + ) + finally: + if operation == "prompt": + self._prompt = None + return await self._reply(result, reply) + + async def _reply(self, result: ProgramResult, reply: str) -> JsonObject: + if result.exit_code: + detail = (result.stderr or result.stdout).strip()[-1000:] + raise RuntimeError( + f"ACP runner request failed (exit {result.exit_code}): {detail}" + ) + response = json.loads(await self.runtime.read(reply)) + if not response.get("ok"): + self._stderr_tail[:] = (await self._runner_log()).encode() + return response + + def memory_state(self) -> dict[str, Any] | None: + state = super().memory_state() + if state is None or self._prompt is None or self._detached is None: + return None + return {**state, **self._prompt, "runner": list(self._detached)} + + async def _continue(self, state: dict[str, Any]) -> ProgramResult: + # The restored box runs the runner and the request's job; only their result + # is still to collect. + self._detached = list(state["runner"]) + async with self._lock: + try: + result = await self.runtime.attach(state["job"]) + response = await self._reply(result, state["reply"]) + except BaseException: + await run_shielded(self._stop(graceful=False)) + raise + return self._finish(response, state["calls_before"]) + async def _request(self, operation: str, **payload: Any) -> JsonObject: + if self._detached is not None: + return await self._call(operation, payload) process = self._process if process is None or self._response_task is None or self._response_task.done(): raise RuntimeError("ACP process is not running") @@ -301,7 +404,7 @@ async def checkpoint_state(self) -> dict[str, Any] | None: # The agent keeps its session in files of the box, so a restored box can load # it in a new process — if the agent advertises `loadSession`. if self._session_id is None: - if self._process is None: + if not self._running: return None response = await self._request("session") result = response.get("result") @@ -324,7 +427,7 @@ async def steer(self, message: str, *, message_id: str | None = None) -> dict: raise RuntimeError("ACP session is closed") if not isinstance(message, str) or not message.strip(): raise ValueError("steering requires a nonempty user message") - if self._process is None: + if not self._running: return {"outcome": "promptRequired", "reason": "noRunningTurn"} response = await self._request("steer", message=message, message_id=message_id) if not response.get("ok"): @@ -380,10 +483,8 @@ async def _run(self, messages: Messages | None) -> ProgramResult: raise HarnessError( f"harness {self.harness.config.id!r} session is already closed" ) - if self._process is None: + if not self._running: await self._start() - assert self._process is not None - assert self._reader is not None calls_before = len(self.trace.calls) try: response = await self._request("prompt", config=config) @@ -394,6 +495,10 @@ async def _run(self, messages: Messages | None) -> ProgramResult: # Only the restored box's first process loads the session; later # turns continue the live one. self._restore_id = None + return self._finish(response, calls_before) + + def _finish(self, response: JsonObject, calls_before: int) -> ProgramResult: + """Consume a prompt request's response as the segment's result.""" turn = _turn_result(response) self.trace.root_reply = turn.reply.strip() self.trace.info["acp_stop_reason"] = turn.stop_reason @@ -417,7 +522,33 @@ async def _run(self, messages: Messages | None) -> ProgramResult: _require_model_turn(self.trace, calls_before, result) return result + async def _stop_detached(self, *, graceful: bool) -> dict[str, Any]: + response_metadata: dict[str, Any] = {} + try: + if graceful: + with contextlib.suppress(BaseException): + response = await asyncio.wait_for(self._call("shutdown", {}), 30) + result = response.get("result") + if response.get("ok") and isinstance(result, dict): + metadata = result.get("response_metadata") + if isinstance(metadata, dict): + response_metadata = metadata + # Best effort: the box may be gone already. + pid = shlex.quote(f"{self._dir}/socket.pid") + with contextlib.suppress(BaseException): + await asyncio.wait_for( + self.runtime.run( + ["sh", "-c", f'kill "$(cat {pid})" 2>/dev/null; true'], {} + ), + 30, + ) + finally: + self._detached = None + return response_metadata + async def _stop(self, *, graceful: bool) -> dict[str, Any]: + if self._detached is not None: + return await self._stop_detached(graceful=graceful) process = self._process response_task = self._response_task stderr_task, self._stderr_task = self._stderr_task, None diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index 5a7ec185af..57b89957d3 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -384,21 +384,20 @@ def write_packet(stream: Any, value: dict) -> None: stream.flush() -async def serve_stream() -> None: - session = ACPSession() - reader = asyncio.StreamReader() - protocol = asyncio.StreamReaderProtocol(reader) - await asyncio.get_running_loop().connect_read_pipe( - lambda: protocol, sys.stdin.buffer - ) - tasks: set[asyncio.Task] = set() - prompt_lock = asyncio.Lock() +class Server: + """Serves the host's session requests, over whichever transport carries them.""" + + def __init__(self) -> None: + self.session = ACPSession() + self.tasks: set[asyncio.Task] = set() + self.prompt_lock = asyncio.Lock() - async def handle(request: dict) -> None: + async def handle(self, request: dict) -> dict: + session = self.session operation = request.get("operation") try: if operation == "prompt": - async with prompt_lock: + async with self.prompt_lock: result = asdict(await session.run(request["config"])) elif operation == "steer": result = await session.steer( @@ -412,6 +411,8 @@ async def handle(request: dict) -> None: ), } elif operation == "shutdown": + if self.tasks: + await asyncio.gather(*self.tasks, return_exceptions=True) result = {"response_metadata": await session.close()} else: raise ValueError(f"unknown ACP session operation: {operation!r}") @@ -425,27 +426,105 @@ async def handle(request: dict) -> None: if operation == "prompt": response["result"] = asdict(session.client.turn_result()) response["id"] = request.get("id") + return response + + def spawn(self, request: dict, reply) -> asyncio.Task: + """Handle `request` in a task of its own and hand its response to `reply`.""" + + async def run() -> None: + reply(await self.handle(request)) + + task = asyncio.create_task(run()) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + return task + + async def close(self) -> None: + for task in self.tasks: + task.cancel() + if self.tasks: + await asyncio.gather(*self.tasks, return_exceptions=True) + await self.session.close() + + +async def serve_stream() -> None: + server = Server() + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await asyncio.get_running_loop().connect_read_pipe( + lambda: protocol, sys.stdin.buffer + ) + + def reply(response: dict) -> None: write_packet(sys.stdout.buffer, response) try: while request := await read_packet(reader): if request.get("operation") == "shutdown": - if tasks: - await asyncio.gather(*tasks) - await handle(request) + reply(await server.handle(request)) break - task = asyncio.create_task(handle(request)) - tasks.add(task) - task.add_done_callback(tasks.discard) + server.spawn(request, reply) finally: - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - await session.close() + await server.close() -async def main() -> None: +async def serve_socket(path: str) -> None: + """Serve session requests on a Unix socket, one per connection, until a shutdown + request: the host's memory snapshots stop the box, which no connection from the + host survives, so each request comes from a job of the box (`call`) instead.""" + server = Server() + stopped = asyncio.Event() + + async def connected( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + try: + request = await read_packet(reader) + if request is None: + return + response = await server.handle(request) + data = json.dumps(response, ensure_ascii=False).encode() + writer.write(len(data).to_bytes(8, "big") + data) + await writer.drain() + if request.get("operation") == "shutdown": + stopped.set() + finally: + writer.close() + + with suppress(FileNotFoundError): + os.unlink(path) + listener = await asyncio.start_unix_server(connected, path, limit=MAX_PACKET_BYTES) + with open(f"{path}.pid", "w") as file: + file.write(f"{os.getpid()}\n") + try: + async with listener: + await stopped.wait() + finally: + await server.close() + + +def call(path: str, request_path: str, reply_path: str) -> None: + """Send the request in `request_path` to the server on the socket at `path` and + write its response to `reply_path`.""" + import socket + + with open(request_path, "rb") as file: + data = file.read() + with socket.socket(socket.AF_UNIX) as connection: + connection.connect(path) + connection.sendall(len(data).to_bytes(8, "big") + data) + stream = connection.makefile("rb") + size = int.from_bytes(stream.read(8), "big") + response = stream.read(size) + if len(response) != size: + raise EOFError("ACP runner closed the connection before its response") + staged = f"{reply_path}.tmp" + with open(staged, "wb") as file: + file.write(response) + os.replace(staged, reply_path) + + +async def main(socket_path: str | None) -> None: task = asyncio.current_task() loop = asyncio.get_running_loop() if task is not None: @@ -453,8 +532,15 @@ async def main() -> None: with suppress(NotImplementedError): loop.add_signal_handler(sig, task.cancel) with suppress(asyncio.CancelledError): - await serve_stream() + if socket_path is None: + await serve_stream() + else: + await serve_socket(socket_path) if __name__ == "__main__": - asyncio.run(main()) + arguments = sys.argv[1:] + if arguments[:1] == ["--call"]: + call(*arguments[1:4]) + else: + asyncio.run(main(arguments[1] if arguments[:1] == ["--socket"] else None)) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index df4be8c0c9..ecec857e9e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -665,6 +665,12 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: backend = backend_cls(self.config.runtime) if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: return config + if ( + config.snapshot == "memory" + and harness.SUPPORTS_MEMORY_RESTORE + and backend.SUPPORTS_MEMORY_SNAPSHOTS + ): + return config key = (harness.config.id, self.config.runtime.type) if key not in _checkpointing_warned: _checkpointing_warned.add(key) diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 7a920b279c..d55c6c901a 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -41,11 +41,20 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): """Rollout checkpointing: snapshot the box every n-th model turn, so a retry - continues from the newest snapshot instead of starting over. The runtime takes the - best snapshot it supports, and each checkpoint records its type. Needs a runtime - that takes snapshots (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new - process can continue the rollout (`Harness.can_restore`); elsewhere a retry starts - over. Every snapshot is deleted once the run is final.""" + continues from the newest snapshot instead of starting over. Each checkpoint + records the type of its snapshot. A memory snapshot holds the box's processes, so + the restored box continues the harness program itself; it needs a runtime that + takes them (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host + (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services + of its own. A filesystem snapshot holds the box's files; it needs a runtime that + takes them (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new process can + continue the rollout (`Harness.can_restore`). Elsewhere a retry starts over. Every + snapshot is deleted once the run is final.""" + + snapshot: Literal["memory", "filesystem"] = "memory" + """The snapshot to take: `memory` where the rollout supports it and a filesystem + snapshot elsewhere, or always `filesystem`. A memory snapshot stops the box for a + few seconds; a filesystem snapshot does not.""" interval: PositiveInt = 10 """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 2ac1fcdc4e..1cd5cf71f9 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -214,6 +214,9 @@ class PrimeConfig(NetworkPolicyConfig): checkpoint: str | None = None """A DURABLE filesystem checkpoint to start from instead of ``image`` (see the SDK's ``checkpoint`` and ``wait_for_checkpoint``).""" + restore_memory: bool = False + """With ``checkpoint``: continue the processes of the paused sandbox that the checkpoint + holds the memory of, instead of booting from its disk.""" idle_timeout: float | None = 3600 """Seconds of inactivity before the sandbox self-deletes (None disables).""" creates_per_min: int | None = None diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 84b7c10f46..ba2eb54bd6 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -15,7 +15,8 @@ SandboxError, boundary, ) -from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.runtimes import Program, ProgramResult, Runtime +from verifiers.v1.runtimes.base import current_program from verifiers.v1.task import TaskData from verifiers.v1.types import Message, Messages, UserMessage from verifiers.v1.utils.decorators import ( @@ -78,6 +79,11 @@ class Harness(ABC, Generic[ConfigT]): leaks host state (auth, config, processes) both ways. Only the minimal in-house loops (`bash`, `null`) override to False.""" + SUPPORTS_MEMORY_RESTORE: ClassVar[bool] = False + """Whether a box restored from a memory snapshot continues this harness: its + program runs as a job of the box that the host only waits for (`Runtime.run(..., + program=)`), and it reaches the host only through the endpoint it was given.""" + @classmethod def can_restore(cls) -> bool: """Whether a new process of this harness can continue a rollout in a box @@ -382,6 +388,13 @@ def __init__( self.data = data self.tool_interception_url = tool_interception_url self._closed = False + self.program = Program() + """The main program's job while a segment runs (`Runtime.run(..., program=)`).""" + self.memory = False + """Whether the rollout takes memory snapshots: the session then keeps no + connection into the box that a stopped box would break.""" + self._attached: dict[str, Any] | None = None + """The state of the segment the restored box still runs (`reattach`).""" async def steer(self, message: str, *, message_id: str | None = None) -> dict: """Deliver a user message during a running turn, if supported by the harness.""" @@ -395,10 +408,34 @@ async def turn(self, messages: Messages | None = None) -> None: raise HarnessError( f"harness {self.harness.config.id!r} session is already closed" ) - async with boundary(HarnessError, f"harness {self.harness.config.id!r}"): - result = await self._run(messages) + attached, self._attached = self._attached, None + token = current_program.set(self.program) + try: + async with boundary(HarnessError, f"harness {self.harness.config.id!r}"): + if attached is not None: + result = await self._continue(attached) + else: + result = await self._run(messages) + finally: + current_program.reset(token) await self.harness._check_result(self.trace, self.runtime, result) + def memory_state(self) -> dict[str, Any] | None: + """What `reattach` needs to continue the running segment in a box restored from + a memory snapshot taken now, or None when no segment runs.""" + if self.program.job is None: + return None + return {"job": dict(self.program.job)} + + def reattach(self, state: dict[str, Any]) -> None: + """Prepare this new session for a box restored from a memory snapshot: its + first `turn` waits for the segment the box still runs instead of starting + one. `state` is what `memory_state` returned when the snapshot was taken.""" + self._attached = state + + async def _continue(self, state: dict[str, Any]) -> ProgramResult: + return await self.runtime.attach(state["job"]) + async def _run(self, messages: Messages | None) -> ProgramResult: if messages is None: kwargs = ( diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 8f6d852144..b9f56996b5 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,6 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_MEMORY_RESTORE = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 503429ddb6..99e93a6d06 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,6 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_MEMORY_RESTORE = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/utils/launch.py b/verifiers/v1/harnesses/utils/launch.py index 786fb67fb3..55d03a287c 100644 --- a/verifiers/v1/harnesses/utils/launch.py +++ b/verifiers/v1/harnesses/utils/launch.py @@ -11,6 +11,7 @@ from verifiers.v1.dialects.chat import message_to_wire from verifiers.v1.harnesses.utils import compaction, core, mcp from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.runtimes.base import current_program from verifiers.v1.trace import Trace from verifiers.v1.types import Messages from verifiers.v1.utils import network @@ -104,5 +105,7 @@ async def launch_chat_program( source, config.resolved_env, activate=activate ) return await runtime.run( - [*program, *args], env if env is not None else {**config.resolved_env} + [*program, *args], + env if env is not None else {**config.resolved_env}, + program=current_program.get(), ) diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 8ce2af2967..f8640de839 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -42,15 +42,27 @@ class Relay: known, so its start overlaps the harness setup; `connect` points it at the interception server, `reroute` moves it, `stop` ends it.""" - def __init__(self, runtime: Runtime, name: str) -> None: + def __init__( + self, runtime: Runtime, name: str, *, running: dict[str, str] | None = None + ) -> None: + """`running` is the `state` of a relay the box already runs (a box restored + from a memory snapshot): nothing is launched, and `connect` moves that relay.""" self.runtime = runtime - self.dir = f"{RELAY_ROOT}/{name}" - self.url = "" + self.dir = f"{RELAY_ROOT}/{name}" if running is None else running["dir"] + self.url = "" if running is None else running["url"] """The base URL the harness reaches the interception server at, once launched.""" self.upstream = "" self.tokens: dict[str, str] = {} self._connected = asyncio.Event() - self._launch = asyncio.create_task(self._start()) + self._launch = asyncio.create_task( + self._start() if running is None else asyncio.sleep(0) + ) + + @property + def state(self) -> dict[str, str]: + """What a later `Relay(..., running=)` needs to take this relay over in a box + restored from a memory snapshot.""" + return {"dir": self.dir, "url": self.url} @property def config_path(self) -> str: diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index cea2fc88c8..328fc0ad6b 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -747,6 +747,7 @@ async def handle_tool(self, request: web.Request) -> web.Response: if session.released: return web.json_response({"error": "rollout concluded"}, status=409) body = from_json(await request.read()) + await session.flowing.wait() try: return web.json_response( await session.decide_tool( @@ -903,6 +904,7 @@ async def handle_request( if idempotent is not None and idempotent.response is not None: logger.debug("Replayed an idempotent request") idempotent.completed_at = now + await session.flowing.wait() return _replay_response(idempotent.response) try: @@ -938,6 +940,7 @@ async def coalesced( response = await asyncio.shield(inflight) if response is None: return _error_response(dialect, "upstream attempt failed", status=503) + await session.flowing.wait() return _replay_response(response) if idempotent.inflight is not None: @@ -1168,6 +1171,8 @@ async def sample() -> web.Response: policy_paths=policy_paths, acp=acp, ) + # A memory snapshot under way must not see this reply reach the box. + await session.flowing.wait() return serve(call_response, events) if streaming: @@ -1304,6 +1309,7 @@ async def handle_state_put(self, request: web.Request) -> web.Response: {"error": f"invalid state PUT for {state_cls.__name__}: {e}"}, status=400, ) + await session.flowing.wait() if session.released: # the trace is sealed — a straggler write must not land return web.json_response({"error": "rollout concluded"}, status=409) session.trace.state = new_state diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 4d7dfdebb9..3288e9c2e8 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -45,7 +45,12 @@ make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms -from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary +from verifiers.v1.session import ( + IdempotentRequest, + RolloutLimits, + RolloutSession, + hook_boundary, +) from verifiers.v1.state import state_cls from verifiers.v1.task import Task, TaskData from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask @@ -81,6 +86,10 @@ class RolloutTimeouts: isolated verifier setup and retries. Finalization has its own budget.""" +REPLAY_SECONDS = 30 +"""A memory checkpoint keeps the replies the server sent this recently: one of them may +not have reached the box when the snapshot stopped it.""" + _HOME_PROBE = ( "home=$(awk -F: -v u=\"$1\" '$1 == u { print $6 }' /etc/passwd); " 'test -n "$home" || exit 1; home=${2:-$home}; echo "$home"; test -d "$home"' @@ -255,6 +264,19 @@ def __init__( response_stops=[fn for boundary, fn in stops if boundary is Response], gates_tools=harness.SUPPORTS_TOOL_INTERCEPTION, ) + self._memory = False + """Whether this run takes memory snapshots: decided at `open`, once the box, + its relay and the task's tool servers are known.""" + self._capturing = asyncio.Lock() + if self._restore is not None and self._restore.snapshot.type == "memory": + # Set before the box boots: its relay repeats the requests that waited as + # soon as it reaches this session. + self._session.pinned_tool_results.update(self._restore.pins) + now = time.monotonic() + for key, (binding, response) in self._restore.replays.items(): + self._session.idempotent_requests[key] = IdempotentRequest( + binding=binding, response=response, completed_at=now + ) self._stack = AsyncExitStack() self._failed = False self._failure: Exception | None = None @@ -365,6 +387,9 @@ async def open(self) -> bool: self.harness.config.name, self.runtime_config.type, ) + memory_restore = ( + self._restore is not None and self._restore.snapshot.type == "memory" + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -409,12 +434,6 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() - if ( - self._checkpointing is not None - and self._borrowed_runtime is None - and runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - ): - self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: isolated = self._scoring_policy.mode == "isolated" self._scoring_task = self.task.grading_task( @@ -464,9 +483,46 @@ async def open(self) -> bool: # A harness in a remote box reaches the interception server through # a relay in its box, which starts alongside the harness setup. if not runtime.is_local: - self._relay = Relay(runtime, self.trace.id) + self._relay = Relay( + runtime, + self.trace.id, + running=self._restore.live["relay"] + if memory_restore + else None, + ) self._stack.push_async_callback(self._relay.stop) - await self.harness.setup(self._agent_runtime) + # A box restored from memory still runs the harness. + if not memory_restore: + await self.harness.setup(self._agent_runtime) + self._memory = ( + self._checkpointing is not None + and self._checkpointing.snapshot == "memory" + and self._borrowed_runtime is None + and runtime.backend.SUPPORTS_MEMORY_SNAPSHOTS + and self.harness.SUPPORTS_MEMORY_RESTORE + # Everything the box reaches the host through must follow it to a + # new box: the relay does, tool servers and services do not. + and self._relay is not None + and not toolsets + and not self._shared_tools + and self.deployment.spec is None + ) + if memory_restore and not self._memory: + raise SandboxError( + "the rollout cannot continue from its memory snapshot" + ) + if ( + self._checkpointing is not None + and self._borrowed_runtime is None + and ( + self._memory + or ( + runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS + and self.harness.can_restore() + ) + ) + ): + self._session.checkpointer = self._checkpoint logger.debug( "Set up %s harness in %s", self.harness.config.name, @@ -501,15 +557,19 @@ async def open(self) -> bool: ) ) harness_base = runtime.host_url(base_url.rstrip("/")).rstrip("/") + # A harness restored from memory still sends the secret it got at its + # launch; the relay swaps it for this session's. + self._secret = ( + self._restore.live["secret"] if memory_restore else model_secret + ) if self._relay is not None: # The harness holds the model secret anyway; the relay gets # nothing more, so reading its config grants no capability. async with boundary(RelayError, "starting the relay"): harness_base = await self._relay.connect( - base_url.rstrip("/"), {model_secret: model_secret} + base_url.rstrip("/"), {self._secret: model_secret} ) self._endpoint = f"{harness_base}/v1" - self._secret = model_secret services = ( { name: self.deployment.target(name) @@ -667,6 +727,7 @@ async def open(self) -> bool: harness_data, **session_kwargs, ) + self._harness_session.memory = self._memory except TimeoutError as e: # Only our expired setup budget owns the timeout; a provider can # raise its own TimeoutError before the overall deadline. @@ -847,6 +908,10 @@ async def restore(self) -> bool: checkpoint = self._restore assert checkpoint is not None and self._harness_data is not None assert self._harness_session is not None + if checkpoint.snapshot.type == "memory": + # The box still runs the segment: wait for it instead of starting one. + self._harness_session.reattach(checkpoint.live["program"]) + return await self.step() # The relaunch re-emits the session's system prompt (the task's, plus a # network notice) through the program's own system message; the tail's copy # would double it. @@ -879,6 +944,8 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: self.restored_turns is not None and turn_number == self.restored_turns + 1 ): return None + if self._memory: + return await self._capture_memory() if len(self.trace._pending) > 1: # Another request is in flight (parallel sub-agents share the trace): the # box is not idle, so no snapshot would match this turn's prefix. @@ -929,6 +996,100 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: ) return checkpoint + async def save(self) -> Checkpoint | None: + """Take a memory checkpoint of the running rollout now, for its owner; None + when the rollout takes none (`RolloutCheckpointConfig`) or runs no segment.""" + if not self._opened or self._closed or not self._memory: + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + return await self._capture_memory() + finally: + self._captures.discard(task) + + async def _capture_memory(self) -> Checkpoint | None: + """Snapshot the box with its memory. Nothing the box asked for is answered + while the snapshot is taken (`RolloutSession.hold`), so the box it stops has + seen exactly the trace as recorded here; a box restored from it asks again + for what was still open.""" + assert self.runtime is not None and self._relay is not None + session = self._session + if self._harness_session is None: + return None + async with self._capturing: + if self._closed: + return None + session.hold() + try: + program = self._harness_session.memory_state() + if program is None: + return None + node = len(self.trace.nodes) - 1 if self.trace.nodes else None + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(session.pinned_tool_results) + now = time.monotonic() + replays = { + key: (request.binding, request.response) + for key, request in session.idempotent_requests.items() + if request.response is not None + and request.completed_at is not None + and now - request.completed_at <= REPLAY_SECONDS + } + started = time.time() + try: + snapshot = await self.runtime.backend.snapshot(memory=True) + except Exception: + logger.warning( + "memory snapshot failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + return None + captured = time.time() + # The box's connections to the server did not survive the stop: its + # relay opens new ones and repeats the requests that wait. + try: + await self._relay.reroute(self._relay.upstream) + except Exception: + logger.warning( + "relay did not reconnect after a memory snapshot (rollout %s)", + self.trace.id, + exc_info=True, + ) + finally: + session.resume() + checkpoint = Checkpoint( + id=snapshot.id, + node=node, + state=state, + info=info, + pins=pins, + replays=replays, + live={ + "program": program, + "relay": self._relay.state, + "secret": self._secret, + }, + snapshot=snapshot, + time=TimeSpan(start=started), + captured=captured, + ) + self.trace.checkpoints.append(checkpoint) + self._durable[checkpoint.id] = asyncio.create_task( + self._make_durable(checkpoint) + ) + logger.debug( + "memory checkpoint %s taken in %s: id=%s node=%s", + checkpoint.id, + format_time(captured - started), + self.trace.id, + node, + ) + return checkpoint + async def _make_durable(self, checkpoint: Checkpoint) -> None: assert self.runtime is not None try: diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 1acfc71a09..8538055967 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -25,6 +25,7 @@ ) from verifiers.v1.runtimes.base import ( BaseRuntimeInfo, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, @@ -103,6 +104,7 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "PrimeBackend", "PrimeConfig", "PrimeRuntimeInfo", + "Program", "ProgramResult", "Runtime", "RuntimeBackend", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 587c678ef7..3d94d1e4c5 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -24,6 +24,7 @@ from verifiers.v1.runtimes.base import ( SERVICE_PORT, BaseRuntimeInfo, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, @@ -55,6 +56,9 @@ """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" _CHECKPOINT_DURABLE_SECONDS = 300 +_PAUSE_SETTLE_SECONDS = 120 +"""How long a pause or a resume gets to take effect; they take seconds.""" +_DEAD_STATUSES = frozenset({"ERROR", "TERMINATED", "TIMEOUT"}) """How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" @@ -157,6 +161,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = True + SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -174,6 +179,10 @@ def __init__(self, config: PrimeConfig, name: str | None = None) -> None: self._expose_network_id: str | None = None self._exposed_ports: list[int] = [] self._expose_lock = asyncio.Lock() + self._awake = asyncio.Event() + """Cleared while a memory snapshot stops the box: the platform refuses every + call into a stopped box, so calls wait for it.""" + self._awake.set() @property def published_port(self) -> int: @@ -213,7 +222,13 @@ async def start(self) -> None: {"mount_path": target, **mount.model_dump()} for target, mount in self.config.mounts.items() ] - if self.config.checkpoint: + memory = bool(self.config.checkpoint and self.config.restore_memory) + if memory: + # The box continues the snapshot's own processes: the platform fixes its + # shape, environment and volumes to the paused box's and refuses others. + source = {"checkpoint_id": self.config.checkpoint, "restore_memory": True} + options = {"timeout_minutes": -1, "idle_timeout_minutes": idle_minutes} + elif self.config.checkpoint: source = {"checkpoint_id": self.config.checkpoint} else: source = { @@ -250,7 +265,7 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - environment_vars=self.env, + **({} if memory else {"environment_vars": self.env}), **source, **{k: v for k, v in options.items() if v is not None}, ) @@ -282,6 +297,9 @@ async def create_and_capture_id(): if self.config.checkpoint else "", ) + if memory: + # The paused box was already prepared; its processes run again. + return await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" ) @@ -297,11 +315,30 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e - async def snapshot(self) -> Snapshot: + async def snapshot(self, *, memory: bool = False) -> Snapshot: # The platform fixes the snapshot's content before the request returns, then # uploads it. Until it is durable the snapshot is stray: the platform refuses # to delete it while this box lives, so teardown frees it after. async def create() -> Snapshot: + if memory: + # A pause is the platform's memory snapshot: it stops the box, and the + # resume continues it. Shielded as one: a box left paused never runs + # its rollout again. + self._awake.clear() + try: + paused = await self._client.pause(self.info.id) + snapshot = Snapshot(id=paused.checkpoint_id, type="memory") + self.stray_snapshots.append(snapshot) + try: + await self._settle("PAUSED", "PAUSE_FAILED") + except BaseException: + self.stray_snapshots.remove(snapshot) + raise + await self._client.resume(self.info.id) + await self._settle("RUNNING", "RESUME_FAILED") + finally: + self._awake.set() + return snapshot checkpoint = await self._client.checkpoint(self.info.id) snapshot = Snapshot(id=checkpoint.id) self.stray_snapshots.append(snapshot) @@ -312,6 +349,33 @@ async def create() -> Snapshot: except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e + def _stopped_by_snapshot(self, error: Exception) -> bool: + """Whether a call failed because a memory snapshot had stopped the box: the + call is safe to repeat once the box runs again.""" + return not self._awake.is_set() or "sandbox is paused" in str(error).lower() + + async def _settle(self, status: str, failure: str) -> None: + """Wait until the platform reports `status` for this box after a pause or a + resume (both return once accepted).""" + deadline = time.monotonic() + _PAUSE_SETTLE_SECONDS + delay = 0.2 + while True: + box = await self._client.get(self.info.id) + if box.status == status: + return + if box.error_type == failure or box.status in _DEAD_STATUSES: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status}: " + f"{box.status} {box.error_type or ''} {box.error_message or ''}" + ) + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status} within " + f"{_PAUSE_SETTLE_SECONDS}s" + ) + await asyncio.sleep(delay) + delay = min(delay * 1.5, 2) + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: started = time.monotonic() try: @@ -348,7 +412,12 @@ async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: @classmethod def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: - return config.model_copy(update={"checkpoint": snapshot.id}) + return config.model_copy( + update={ + "checkpoint": snapshot.id, + "restore_memory": snapshot.type == "memory", + } + ) async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None @@ -398,8 +467,14 @@ async def prepare_execution( ) async def run( - self, runtime: Runtime, argv: list[str], env: dict[str, str] + self, + runtime: Runtime, + argv: list[str], + env: dict[str, str], + *, + program: Program | None = None, ) -> ProgramResult: + await self._awake.wait() try: # Poll directly so rollout cancellation owns the execution timeout. job = await self._client.start_background_job( @@ -409,13 +484,41 @@ async def run( env=runtime.process_env(env), user=runtime.user, ) + except ( + Exception + ) as e: # a sandbox/API failure is one rollout's problem, not the eval's + raise SandboxError(f"prime exec failed: {e}") from e + if program is not None: + program.job = job.model_dump() + try: + return await self._wait_job(job) + finally: + if program is not None: + program.job = None + + async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: + from prime_sandboxes import BackgroundJob + + # The job's files and process came with the snapshot; only the box is new. + return await self._wait_job( + BackgroundJob(**{**job, "sandbox_id": self.info.id}) + ) + + async def _wait_job(self, job: Any) -> ProgramResult: + try: delay = 0.1 output_retries = 0 missing = None while True: - result = await self._client.get_background_job( - self.info.id, job, timeout=_OUTPUT_DEADLINE_SECONDS - ) + await self._awake.wait() + try: + result = await self._client.get_background_job( + self.info.id, job, timeout=_OUTPUT_DEADLINE_SECONDS + ) + except Exception as e: + if not self._stopped_by_snapshot(e): + raise + continue # Under load the SDK can see a job finish yet miss its output (its bounded # output reads expire while queued) and reports that as `*_error`. The job # is done, so asking again re-reads the output. @@ -494,6 +597,7 @@ async def create_and_capture_id() -> None: async def run_background( self, runtime: Runtime, argv: list[str], env: dict[str, str], log: str ) -> None: + await self._awake.wait() # A server-owned job runs without a client stream for its whole life. command = f"exec {shlex.join(argv)} > {shlex.quote(log)} 2>&1 < /dev/null" try: @@ -510,6 +614,7 @@ async def run_background( async def read( self, runtime: Runtime, path: str, max_bytes: int | None = None ) -> bytes: + await self._awake.wait() # The file API acts as the default user; a user view reads through exec. if max_bytes is not None or runtime.user is not None: reader = "cat" if max_bytes is None else f"head -c {max_bytes}" @@ -558,6 +663,7 @@ async def read( raise SandboxError(f"read {path!r}: {e}") from e async def write(self, runtime: Runtime, path: str, data: bytes) -> None: + await self._awake.wait() # The gateway creates missing parents and uploads binary data without command-line # limits. Resolve relative paths here because uploads do not use this runtime's workdir. target = ( diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 88cafda050..7e7b185ae9 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -12,6 +12,7 @@ import weakref from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping +from contextvars import ContextVar from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar, Literal @@ -149,9 +150,27 @@ class Snapshot(BaseModel): id: str """The provider's id for the snapshot.""" - type: Literal["filesystem"] = "filesystem" + type: Literal["filesystem", "memory"] = "filesystem" """What the snapshot captures, as the runtime took it: `filesystem` holds the box's - files, not its processes.""" + files, not its processes; `memory` holds its files and its memory, so a box that + boots from it continues every process where the snapshot stopped it.""" + + +@dataclass +class Program: + """The background job of a harness's main program, as its runtime started it + (`Runtime.run(..., program=)`). A box that boots from a memory snapshot still runs + the job, and `Runtime.attach` waits for it there.""" + + job: dict[str, Any] | None = None + """The provider's handle of the job while it runs; None before and after.""" + + +current_program: ContextVar[Program | None] = ContextVar( + "vf_current_program", default=None +) +"""The `Program` of the harness segment this task runs (`HarnessSession.turn`), for the +launch that runs the segment's main program.""" @dataclass @@ -208,6 +227,10 @@ class RuntimeBackend(ABC): (`restore_config`). Rollout checkpointing (`AgentConfig.checkpoint`) takes one at model turns, so a retry continues from there instead of starting over.""" + SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = False + """Whether `snapshot(memory=True)` captures the box's memory with its filesystem, so + a new box continues the processes of the old one (`attach`).""" + info: BaseRuntimeInfo def __init__(self, name: str | None = None) -> None: @@ -245,12 +268,21 @@ def type(self) -> str: async def start(self) -> None: pass - async def snapshot(self) -> Snapshot: + async def snapshot(self, *, memory: bool = False) -> Snapshot: """Snapshot the box and return it once its content is fixed: writes after this returns are not in it. It becomes restorable only later (`wait_snapshot`); the - box keeps running.""" + box keeps running. With `memory` the snapshot holds the box's memory too: the + box stops while it is taken and runs again when this returns, and its + connections to the outside may be dead.""" raise NotImplementedError(f"{type(self).__name__} does not support snapshots") + async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: + """Wait for a job that a box started before the memory snapshot this box booted + from (`Program.job`), and return its result like `run`.""" + raise NotImplementedError( + f"{type(self).__name__} does not support memory snapshots" + ) + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: """Wait until `snapshot` is restorable and return its stored size in bytes, when the provider reports it. Raises if it never becomes restorable. The box diff --git a/verifiers/v1/runtimes/runtime.py b/verifiers/v1/runtimes/runtime.py index a7965fa0a6..3f961d3988 100644 --- a/verifiers/v1/runtimes/runtime.py +++ b/verifiers/v1/runtimes/runtime.py @@ -11,13 +11,14 @@ from dataclasses import dataclass, field, replace from pathlib import PurePosixPath from types import MappingProxyType -from typing import Self +from typing import Any, Self from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes.base import ( _ENSURE_UV, _INSTALL_CURL, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, @@ -112,10 +113,21 @@ async def set_network(self, network: NetworkPolicyConfig) -> None: ) await self.backend.prepare_execution([], policy) - async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: - """Run `argv` to completion and collect its output.""" + async def run( + self, argv: list[str], env: dict[str, str], *, program: Program | None = None + ) -> ProgramResult: + """Run `argv` to completion and collect its output. `program` marks the run as + a harness's main program: a runtime that takes memory snapshots records the + job there while it runs, for `attach`.""" + if program is not None and self.backend.SUPPORTS_MEMORY_SNAPSHOTS: + return await self.backend.run(self, argv, env, program=program) return await self.backend.run(self, argv, env) + async def attach(self, job: dict[str, Any]) -> ProgramResult: + """Wait for the job a box started before the memory snapshot this box booted + from, and return its result like `run`.""" + return await self.backend.attach(self, job) + async def exec(self, argv: list[str], env: dict[str, str]) -> RuntimeProcess: """Start `argv` and return a handle that streams its stdio.""" return await self.backend.exec(self, argv, env) diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 51d3b72117..aa98addd49 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -234,6 +234,21 @@ class RolloutSession: the reply waits for the capture (`Rollout` sets it when its runtime supports checkpoints and a retry may restore one). Returns the checkpoint it took, if any.""" + flowing: asyncio.Event = field(default_factory=asyncio.Event, repr=False) + """Cleared while a memory snapshot of the box is taken (`hold`): nothing the box + asked for is answered or written until it is set again, so the box the snapshot + stops has seen exactly what the trace held when the hold began.""" + + def __post_init__(self) -> None: + self.flowing.set() + + def hold(self) -> None: + """Stop answering the box: replies, tool verdicts and state writes wait.""" + self.flowing.clear() + + def resume(self) -> None: + self.flowing.set() + @property def stopped(self) -> bool: return self.trace.stop_condition is not None diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index fc97735422..de4021e897 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -400,6 +400,14 @@ class Checkpoint(BaseModel): """The rollout's pinned tool results as the snapshot was taken (`RolloutSession.pinned_tool_results`): a restored harness re-sends its own copies, which must be pinned again to match the trace. Live-only.""" + live: dict[str, Any] = Field(default_factory=dict, exclude=True) + """What continues the box of a memory snapshot: the harness program's job + (`HarnessSession.memory_state`), the box's relay and the secret the harness holds. + Live-only.""" + replays: dict[str, Any] = Field(default_factory=dict, exclude=True) + """The replies the server had just sent when a memory snapshot stopped the box + (`RolloutSession.idempotent_requests`): the restored box asks again for one that had + not arrived, and gets the reply the trace already holds. Live-only.""" time: TimeSpan = Field(default_factory=TimeSpan) """From the snapshot request until it was durable (restorable); `end` stays 0 while it is not.""" From 7a520d933a6d3d8c7b991405ee49366f8073d348 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:24:01 +0000 Subject: [PATCH 23/53] save running rollouts as restore points for another process Co-Authored-By: Claude --- verifiers/v1/agent.py | 41 +++++-- verifiers/v1/env.py | 13 ++- verifiers/v1/restore.py | 136 ++++++++++++++++++++++++ verifiers/v1/rollout.py | 37 +++++-- verifiers/v1/runtimes/backends/prime.py | 3 +- verifiers/v1/runtimes/base.py | 6 ++ verifiers/v1/trace.py | 5 +- 7 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 verifiers/v1/restore.py diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index ecec857e9e..8c3d2e59a3 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -35,6 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer +from verifiers.v1.restore import RestorePoint, restoring from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, @@ -712,6 +713,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: RestorePoint | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -727,7 +729,10 @@ async def run( checkpoint whose restore failed is skipped, and without one the retry starts over. Every attempt's trace keeps the run's id with its own `attempt`, and the final trace keeps every attempt's errors. `on_score(trace, runtime)` runs after task scoring in - the selected verifier workspace, inside its timeout and retry boundary.""" + the selected verifier workspace, inside its timeout and retry boundary. + `restore` continues a rollout another process saved (`save_all`) instead of + starting one: its trace, in a new box from its memory checkpoint; the episode's + restore point (`RunSlot.restore`) is taken the same way.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries @@ -737,6 +742,23 @@ async def run( history: list[Error] = [] # errors of the attempts started over from trace: Trace | None = None target: Checkpoint | None = None # the checkpoint the next attempt restores + first_attempt = 0 # the attempts the trace made before this run + pending = restoring.get() + if restore is None and pending: + restore = pending.pop() + if restore is not None: + if checkpointing is None: + logger.warning( + "Starting rollout %s over: this agent takes no checkpoints, so it " + "cannot continue from its restore point", + restore.id, + ) + else: + trace, target = restore.open(task) + trace_id = trace.id + first_attempt = trace.attempt + if on_trace is not None: + on_trace(trace) snapshots: dict[str, Snapshot] = {} # every box snapshot any attempt took deleted: set[str] = set() failed_restores: set[str] = set() @@ -751,14 +773,18 @@ async def run( with scope(trace=short_id(trace_id), agent=self._log_name): try: while True: - restoring = ( + restoring_from = ( (trace, target) if trace is not None and target is not None else None ) # This attempt's own errors decide — a restored trace still carries # the earlier attempts'. - seen = len(restoring[0].errors) if restoring is not None else 0 + seen = ( + len(restoring_from[0].errors) + if restoring_from is not None + else 0 + ) run = await self._run_once( task, runtime, @@ -767,14 +793,17 @@ async def run( collect_artifacts, on_score, trace_id=trace_id, - attempt=attempt + 1, - restore=restoring, + attempt=first_attempt + attempt + 1, + restore=restoring_from, checkpoint=checkpointing, ) trace = run.trace snapshots.update((s.id, s) for s in run.rewound_snapshots) + # An owner's snapshots (`save_all`) are the owner's to delete. snapshots.update( - (c.snapshot.id, c.snapshot) for c in trace.checkpoints + (c.snapshot.id, c.snapshot) + for c in trace.checkpoints + if not c.pinned ) deleted.update(run.deleted_snapshots) if target is not None and ( diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1c58f9af84..a51dbb44de 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -30,6 +30,7 @@ requires_tunnel, ) from verifiers.v1.mcp import SharedToolServer, serve_shared +from verifiers.v1.restore import RestorePoint, restoring from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task from verifiers.v1.trace import Error, Trace, TraceTask @@ -67,6 +68,9 @@ class RunSlot: episode: Episode | None = None done: bool = False started: float | None = None + restore: RestorePoint | None = None + """A rollout another process saved (`save_all`): the episode's first attempt + continues it instead of starting one.""" @classmethod def finished(cls, episode: Episode) -> "RunSlot": @@ -246,6 +250,7 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, + restore: RestorePoint | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -273,7 +278,11 @@ async def run_episode( async with boundary(EnvError, f"{type(self).__name__}.setup()"): await self.setup(agents) async with boundary(EnvError, f"{type(self).__name__}.run()"): - await self.run(task, agents) + token = restoring.set([restore] if restore is not None else None) + try: + await self.run(task, agents) + finally: + restoring.reset(token) if not episode.traces: raise ValueError( f"{type(self).__name__}.run() ran no agent — every " @@ -370,6 +379,8 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, + # A retry of the episode starts over. + restore=slot.restore if attempts == 1 else None, ) started: float | None = None diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py new file mode 100644 index 0000000000..f0fb8ce64e --- /dev/null +++ b/verifiers/v1/restore.py @@ -0,0 +1,136 @@ +"""Restore points: a running rollout's memory checkpoint in a form another process can +continue. + +A retry restores a checkpoint in the process that took it, from the live `Checkpoint`. +An owner that outlives its process (a training run that stops and resumes) saves every +running rollout instead (`save_all`): each one takes a memory checkpoint now and +returns a `RestorePoint`, made of plain data the owner can store. A later process hands +the point back (`RunSlot.restore`, `Agent.run(restore=)`), and the rollout continues in +a new box from the checkpoint: its trace up to there, and the harness program the box +still runs. + +The owner holds the snapshot of a restore point (`Checkpoint.pinned`): no rollout +deletes it, and the owner deletes it once nothing will restore it +(`RuntimeBackend.discard`). +""" + +from __future__ import annotations + +import asyncio +import weakref +from contextvars import ContextVar +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from pydantic import TypeAdapter + +from verifiers.v1.session import ReplayResponse +from verifiers.v1.state import state_cls +from verifiers.v1.trace import Checkpoint, Trace +from verifiers.v1.types import ToolMessage +from verifiers.v1.utils.logging import get_logger + +if TYPE_CHECKING: + from verifiers.v1.rollout import Rollout + from verifiers.v1.task import Task + +logger = get_logger(__name__) + +LIVE: weakref.WeakSet[Rollout] = weakref.WeakSet() +"""The rollouts of this process that run a box (between `open` and `close`).""" + +restoring: ContextVar[list[RestorePoint] | None] = ContextVar( + "vf_restoring", default=None +) +"""The restore point the episode this task runs continues from (`Env.run_episode`), in +a list its first agent run takes it out of.""" + + +@dataclass +class RestorePoint: + """A rollout as a memory checkpoint left it, as plain data: every field survives a + pickle or a msgpack round trip with raw bytes and numpy arrays.""" + + trace: dict[str, Any] + """The trace when the checkpoint was taken, with its token arrays + (`Trace.model_dump(context={"token_arrays": True})`).""" + checkpoint: str + """The id of the checkpoint on `trace` to continue from.""" + state: Any + """The trace's typed state, as JSON data.""" + info: dict[str, Any] + pins: list[tuple[str, str, dict[str, Any]]] + """The pinned tool results: conversation prefix, call id and the message.""" + replays: dict[str, tuple[str, bytes, int, bytes, str]] + """The replies the box may ask for again: request path and digest, then the reply's + status, body and content type.""" + live: dict[str, Any] + """`Checkpoint.live`.""" + + @property + def id(self) -> str: + """The trace's id.""" + return self.trace["id"] + + @property + def snapshot(self) -> dict[str, Any]: + """The box snapshot the point continues from (`Snapshot` as data).""" + for checkpoint in self.trace["checkpoints"]: + if checkpoint["id"] == self.checkpoint: + return checkpoint["snapshot"] + raise ValueError(f"restore point {self.id} lost its checkpoint") + + @classmethod + def of(cls, trace: Trace, checkpoint: Checkpoint) -> RestorePoint: + return cls( + trace=trace.model_dump(context={"token_arrays": True}), + checkpoint=checkpoint.id, + state=TypeAdapter(type(checkpoint.state)).dump_python( + checkpoint.state, mode="json" + ), + info=dict(checkpoint.info or {}), + pins=[ + (prefix, call_id, message.model_dump(mode="json")) + for (prefix, call_id), message in checkpoint.pins.items() + ], + replays={ + key: (path, digest, reply.status, reply.body, reply.content_type) + for key, ((path, digest), reply) in checkpoint.replays.items() + }, + live=checkpoint.live, + ) + + def open(self, task: Task) -> tuple[Trace, Checkpoint]: + """The trace and its live checkpoint, for `task`'s rollout to restore.""" + trace = Trace.model_validate(self.trace) + checkpoint = next(c for c in trace.checkpoints if c.id == self.checkpoint) + trace.state = state_cls(type(task)).model_validate(self.state) + checkpoint.state = trace.state.model_copy(deep=True) + checkpoint.info = dict(self.info) + checkpoint.pins = { + (prefix, call_id): ToolMessage.model_validate(message) + for prefix, call_id, message in self.pins + } + checkpoint.replays = { + key: ((path, digest), ReplayResponse(status, body, content_type)) + for key, (path, digest, status, body, content_type) in self.replays.items() + } + checkpoint.live = self.live + return trace, checkpoint + + +async def save_all() -> list[RestorePoint]: + """Take a memory checkpoint of every running rollout of this process and return + their restore points. A rollout that takes no memory snapshots, runs no harness + segment right now, or whose snapshot fails has none: it starts over when its owner + runs it again. The snapshots are pinned: the caller owns them.""" + + async def save(rollout: Rollout) -> RestorePoint | None: + try: + return await rollout.save() + except Exception: + logger.warning("saving rollout %s failed", rollout.trace.id, exc_info=True) + return None + + points = await asyncio.gather(*(save(rollout) for rollout in list(LIVE))) + return [point for point in points if point is not None] diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 3288e9c2e8..645340d6ca 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -45,6 +45,7 @@ make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms +from verifiers.v1.restore import LIVE, RestorePoint from verifiers.v1.session import ( IdempotentRequest, RolloutLimits, @@ -131,7 +132,7 @@ def kept(checkpoints: list[Checkpoint], keep: Literal["last"] | int) -> set[str] live = [ (index, c) for index, c in enumerate(checkpoints) - if c.time.end and not c.deleted + if c.time.end and not c.deleted and not c.pinned ] if not live: return set() @@ -346,6 +347,7 @@ async def open(self) -> bool: interception slot and tool servers. Returns whether the exchange can proceed; a setup failure is captured onto the trace.""" self._opened = True + LIVE.add(self) self.trace.timing.boot.start = time.time() self.trace.notify() if self._borrowed_runtime is None: @@ -996,20 +998,33 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: ) return checkpoint - async def save(self) -> Checkpoint | None: - """Take a memory checkpoint of the running rollout now, for its owner; None - when the rollout takes none (`RolloutCheckpointConfig`) or runs no segment.""" + async def save(self) -> RestorePoint | None: + """Take a memory checkpoint of the running rollout now and return its restore + point, for an owner that continues the rollout in another process + (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`). None when + the rollout takes no memory snapshots (`RolloutCheckpointConfig`), runs no + segment, or the snapshot did not become restorable.""" if not self._opened or self._closed or not self._memory: return None task = asyncio.current_task() if task is not None: self._captures.add(task) try: - return await self._capture_memory() + checkpoint = await self._capture_memory(pinned=True) finally: self._captures.discard(task) + if checkpoint is None: + return None + # Before the next await: the trace moves on once replies flow again. + point = RestorePoint.of(self.trace, checkpoint) + durable = self._durable.get(checkpoint.id) + if durable is not None: + await asyncio.wait({durable}) + if not checkpoint.time.end: + return None + return point - async def _capture_memory(self) -> Checkpoint | None: + async def _capture_memory(self, *, pinned: bool = False) -> Checkpoint | None: """Snapshot the box with its memory. Nothing the box asked for is answered while the snapshot is taken (`RolloutSession.hold`), so the box it stops has seen exactly the trace as recorded here; a box restored from it asks again @@ -1076,6 +1091,7 @@ async def _capture_memory(self) -> Checkpoint | None: snapshot=snapshot, time=TimeSpan(start=started), captured=captured, + pinned=pinned, ) self.trace.checkpoints.append(checkpoint) self._durable[checkpoint.id] = asyncio.create_task( @@ -1124,7 +1140,10 @@ async def _prune(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if c.time.end and not c.deleted and c.id not in keep + if c.time.end + and not c.deleted + and not c.pinned + and c.id not in keep ), *( s @@ -1178,6 +1197,7 @@ async def abort(self) -> None: (a cancellation mid-setup, a lifetime bug raised to the caller) means the driver will never reach `close()`. Safe after a partial `close()`.""" self._closed = True + LIVE.discard(self) await self._settle_checkpoints(wait=False) if self._harness_session is not None: with contextlib.suppress(Exception): @@ -1207,7 +1227,7 @@ async def abort(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if not c.deleted + if not c.deleted and not c.pinned ), *self.runtime.backend.stray_snapshots, ] @@ -1305,6 +1325,7 @@ async def close(self) -> Trace: if self._closed: return self.trace self._closed = True + LIVE.discard(self) trace = self.trace skip_reason = self._scoring_skip_reason() runtime = self.runtime diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 3d94d1e4c5..1a2b21a8b9 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -393,7 +393,8 @@ async def wait_snapshot(self, snapshot: Snapshot) -> int | None: ) return durable.stored_bytes - async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + @classmethod + async def discard(cls, snapshots: list[Snapshot]) -> list[str]: # The platform refuses only the newest checkpoint of a running box (the parent # of its live state); older ones go while it runs, their data kept as long as a # newer one needs it. Usually called after teardown released `_client`. diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 7e7b185ae9..e0125d7cb0 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -293,6 +293,12 @@ async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: """Free snapshots nothing will restore from and return the ids it freed. Best-effort: a provider may refuse one a live box still needs (prime: the newest one of a running box).""" + return await self.discard(snapshots) + + @classmethod + async def discard(cls, snapshots: list[Snapshot]) -> list[str]: + """`discard_snapshots` without a box: for an owner that holds snapshots past + the boxes that took them (`Checkpoint.pinned`).""" return [] @classmethod diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index de4021e897..6429e83841 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -423,6 +423,9 @@ class Checkpoint(BaseModel): (`HarnessSession.checkpoint_state`), e.g. an ACP agent's session id.""" restored: bool = False """Whether a retry restored the run from this snapshot.""" + pinned: bool = False + """Whether the rollout's owner holds the snapshot (`save_all`): no rollout deletes + it, the owner does.""" deleted: bool = False """Whether the snapshot was deleted (by `RolloutCheckpointConfig.keep`, or once the run is final): it can no longer be restored.""" @@ -1093,7 +1096,7 @@ def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: position = ids.index(checkpoint.id) + 1 rewound = self.checkpoints[position:] del self.checkpoints[position:] - dropped = [c.snapshot for c in rewound] + dropped = [c.snapshot for c in rewound if not c.pinned] self.rewards = {} self.metrics = {} self.dropped = None From 14fa9568cb6cc284d7db4accf377d7d9e2cd06f8 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:32:34 +0000 Subject: [PATCH 24/53] test a saved rollout continued by another owner Co-Authored-By: Claude --- .../integration/fixtures/memory_restore_v1.py | 7 +- tests/integration/test_memory_checkpoint.py | 76 ++++++++++++++++++- verifiers/v1/restore.py | 10 ++- verifiers/v1/rollout.py | 13 +++- 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/tests/integration/fixtures/memory_restore_v1.py b/tests/integration/fixtures/memory_restore_v1.py index 3891c6bfd4..87884d5dfe 100644 --- a/tests/integration/fixtures/memory_restore_v1.py +++ b/tests/integration/fixtures/memory_restore_v1.py @@ -23,8 +23,11 @@ STEP_ONE = "step1.txt" DONE = "done.txt" READ = f"timeout 10 cat {FIFO}" -# The nonce is the script's first argument: it lives in the process, not in a file. -SERVER = f'rm -f {FIFO}; mkfifo {FIFO}; while :; do printf %s "$1" > {FIFO}; done' +# The nonce is the script's first argument: it lives in the process, not in a file. The +# sleep lets a reader see the end of one copy before the next write opens the FIFO. +SERVER = ( + f'rm -f {FIFO}; mkfifo {FIFO}; while :; do printf %s "$1" > {FIFO}; sleep 0.5; done' +) class MemoryRestoreTaskConfig(vf.TaskConfig): diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py index 800ba5b6a5..4368b42d4b 100644 --- a/tests/integration/test_memory_checkpoint.py +++ b/tests/integration/test_memory_checkpoint.py @@ -8,14 +8,21 @@ a box that booted from the snapshot's files alone would fail. Every case checks that the run leaks no snapshot.""" +import asyncio +import pickle + import pytest +from conftest import _eval_config from grid import grid from prime_sandboxes import AsyncSandboxClient +from verifiers.v1.clients import ModelContext +from verifiers.v1.env import RunSlot from verifiers.v1.harness import RESTORE_PROMPT -from verifiers.v1.runtimes import PrimeBackend +from verifiers.v1.restore import save_all +from verifiers.v1.runtimes import PrimeBackend, Snapshot +from verifiers.v1.utils.loaders import harness_class, load_environment from verifiers.v1.types import content_text -from verifiers.v1.utils.loaders import harness_class RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} NEEDS = ("memory",) @@ -132,3 +139,68 @@ async def test_filesystem_snapshots_stay_selectable( assert {kind for _, kind in taken} == {"filesystem"} assert trace.reward == 0.0 await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True +) +async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_runtime): + """`save_all` checkpoints a running rollout for its owner. The rollout is then + cancelled with everything it holds, and its restore point goes through a pickle. A + second env, with a server, a tunnel and secrets of its own, continues the rollout + from the point: in a new box, with the harness program and the task's process + still running.""" + config = _eval_config( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": 99}}, + max_turns=12, + max_tokens=4096, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + first = load_environment(config.env) + (task,) = list(first.taskset) + async with first.serving(): + (slot,) = first.slots(task) + running = asyncio.ensure_future(first.run_slot(slot, ctx)) + try: + # Past its first turn: the box holds work the trace recorded. + async with asyncio.timeout(600): + while not (slot.traces and slot.traces[0].num_turns >= 1): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + turns_saved = len([node for node in point.trace["nodes"] if node["sampled"]]) + point = pickle.loads(pickle.dumps(point)) + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point.id + assert trace.reward == 1.0 + assert trace.metrics["copied"] == 1.0, [ + (n.message.role, n.sampled, str(n.message.model_dump(exclude_none=True))[:600]) + for n in trace.nodes + ] + assert trace.has_restored and trace.attempt == 2 + assert trace.num_turns > turns_saved >= 1 + assert sum(node.parent is None for node in trace.nodes) == 1 + # The owner holds the snapshot: no rollout deleted it. + (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] + assert snapshot_id == point.checkpoint + async with AsyncSandboxClient() as client: + assert (await client.get_checkpoint(snapshot_id)).state == "DURABLE" + assert await PrimeBackend.discard([Snapshot(id=snapshot_id, type="memory")]) == [ + snapshot_id + ] + await assert_no_leaks(taken) diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py index f0fb8ce64e..752767c534 100644 --- a/verifiers/v1/restore.py +++ b/verifiers/v1/restore.py @@ -132,5 +132,11 @@ async def save(rollout: Rollout) -> RestorePoint | None: logger.warning("saving rollout %s failed", rollout.trace.id, exc_info=True) return None - points = await asyncio.gather(*(save(rollout) for rollout in list(LIVE))) - return [point for point in points if point is not None] + rollouts = list(LIVE) + points = [ + point + for point in await asyncio.gather(*(save(rollout) for rollout in rollouts)) + if point is not None + ] + logger.info("Saved %d of %d running rollouts", len(points), len(rollouts)) + return points diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 645340d6ca..9c7ea2b811 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -1005,6 +1005,12 @@ async def save(self) -> RestorePoint | None: the rollout takes no memory snapshots (`RolloutCheckpointConfig`), runs no segment, or the snapshot did not become restorable.""" if not self._opened or self._closed or not self._memory: + logger.debug( + "Not saving rollout %s: memory=%s closed=%s", + self.trace.id, + self._memory, + self._closed, + ) return None task = asyncio.current_task() if task is not None: @@ -1169,8 +1175,11 @@ async def _settle_checkpoints(self, *, wait: bool) -> None: await asyncio.gather(*self._captures, return_exceptions=True) tasks = list(self._durable.values()) if not wait: - for task in tasks: - task.cancel() + # An owner waits for its snapshots whatever becomes of this run (`save`). + pinned = {c.id for c in self.trace.checkpoints if c.pinned} + for checkpoint_id, task in self._durable.items(): + if checkpoint_id not in pinned: + task.cancel() await asyncio.gather(*tasks, return_exceptions=True) # A task cancelled before it first ran never reached its own cleanup. for checkpoint_id, task in list(self._durable.items()): From addc9d7343c2485fee54755b70a788542469a77b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:38:35 +0000 Subject: [PATCH 25/53] share batched status polls between pausing boxes Co-Authored-By: Claude --- verifiers/v1/runtimes/backends/prime.py | 61 +++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 1a2b21a8b9..bb84d7a45d 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -59,6 +59,7 @@ _PAUSE_SETTLE_SECONDS = 120 """How long a pause or a resume gets to take effect; they take seconds.""" _DEAD_STATUSES = frozenset({"ERROR", "TERMINATED", "TIMEOUT"}) +_STATUS_POLL_SECONDS = 0.3 """How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" @@ -70,10 +71,62 @@ BASE_LABELS: list[str] = [] +class _StatusPoller: + """Batched status lookups for the boxes that wait for a pause or a resume to take + effect: one request serves up to 100 boxes, so a save of every rollout of a run + costs a few requests per poll instead of one per box.""" + + def __init__(self, client: Any) -> None: + self.client = client + self.waiting: dict[str, asyncio.Future] = {} + self.task: asyncio.Task | None = None + + async def next(self, sandbox_id: str) -> Any: + """The box's status at the next poll (one waiter per box).""" + future = asyncio.get_running_loop().create_future() + self.waiting[sandbox_id] = future + if self.task is None or self.task.done(): + self.task = asyncio.create_task(self._run()) + return await future + + async def _run(self) -> None: + while self.waiting: + await asyncio.sleep(_STATUS_POLL_SECONDS) + waiting, self.waiting = self.waiting, {} + ids = list(waiting) + await asyncio.gather( + *( + self._poll({i: waiting[i] for i in ids[start : start + 100]}) + for start in range(0, len(ids), 100) + ) + ) + + async def _poll(self, waiting: dict[str, asyncio.Future]) -> None: + try: + response = await self.client.get_sandbox_statuses(list(waiting)) + except Exception as error: # noqa: BLE001 - each waiter reports it + results: dict[str, Any] = dict.fromkeys(waiting, error) + else: + results = {status.sandbox_id: status for status in response.statuses} + results.update( + (item.sandbox_id, SandboxError(f"{item.code}: {item.message}")) + for item in response.errors + ) + for sandbox_id, future in waiting.items(): + result = results.get(sandbox_id, SandboxError("no status returned")) + if future.done(): + continue + if isinstance(result, Exception): + future.set_exception(result) + else: + future.set_result(result) + + @dataclass class _SharedClient: client: Any leases: int = 0 + poller: _StatusPoller | None = None # One `AsyncSandboxClient` per event loop, leased by every live runtime on it. The SDK @@ -358,9 +411,11 @@ async def _settle(self, status: str, failure: str) -> None: """Wait until the platform reports `status` for this box after a pause or a resume (both return once accepted).""" deadline = time.monotonic() + _PAUSE_SETTLE_SECONDS - delay = 0.2 + shared = _shared_clients[asyncio.get_running_loop()] + if shared.poller is None: + shared.poller = _StatusPoller(shared.client) while True: - box = await self._client.get(self.info.id) + box = await shared.poller.next(self.info.id) if box.status == status: return if box.error_type == failure or box.status in _DEAD_STATUSES: @@ -373,8 +428,6 @@ async def _settle(self, status: str, failure: str) -> None: f"prime sandbox {self.info.id} did not reach {status} within " f"{_PAUSE_SETTLE_SECONDS}s" ) - await asyncio.sleep(delay) - delay = min(delay * 1.5, 2) async def wait_snapshot(self, snapshot: Snapshot) -> int | None: started = time.monotonic() From 8ba38c819f99f0c6da4c50755515bae778c50b6e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:40:23 +0000 Subject: [PATCH 26/53] test and document memory checkpoints Co-Authored-By: Claude --- .github/workflows/tests.yml | 2 + docs/agent.md | 10 +- docs/architecture.md | 2 + docs/harnesses.md | 1 + tests/unit/test_checkpoint.py | 170 +++++++++++++++++++++++++++++++++- 5 files changed, 181 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 19409ffe18..f3caea2975 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,6 +99,8 @@ jobs: tests: tests/integration/test_resume_tool.py - behavior: Checkpoint Restore tests: tests/integration/test_checkpoint.py + - behavior: Memory Checkpoint Restore + tests: tests/integration/test_memory_checkpoint.py - behavior: User Sim tests: tests/integration/test_user_sim.py - behavior: Agent User diff --git a/docs/agent.md b/docs/agent.md index 8aabc1e8aa..24c0730d20 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,10 +77,18 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). Elsewhere a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. Each checkpoint records the type of its snapshot: + +- A **memory** snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. It needs a runtime that takes memory snapshots (prime), a harness that runs detached from the host (`SUPPORTS_MEMORY_RESTORE`: null, bash and the ACP harnesses), and a rollout without tool servers or services of its own. +- A **filesystem** snapshot holds the box's files. The retry starts a new harness process in the restored box and tells it that its processes are gone. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). + +A rollout takes memory snapshots where it can and filesystem snapshots elsewhere. Without either, a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] +snapshot = "memory" # or "filesystem": never stop the box for a snapshot interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` + +An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns a `RestorePoint` for each: plain data the owner stores. A later process continues a rollout from its point with `RunSlot(task, id=..., restore=point)` (or `Agent.run(task, restore=point)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. diff --git a/docs/architecture.md b/docs/architecture.md index c685ad35b9..9b2d55ca0c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,6 +32,8 @@ A Prime tunnel can drop for a while (its client reconnects) or for good (the tun - While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. - When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. +A box restored from a memory snapshot (`agent.checkpoint`) still runs its relay. The new rollout points that relay at its own interception server and maps the secret the harness holds to the new session's secret, so the harness keeps its URL and its secret. Model calls that were open when the snapshot stopped the box go to the new server. + A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a uv script, like the harness programs and tool servers verifiers already runs in the box. A relay that does not start, gives up on a request or exits fails the rollout with a `RelayError`. ## Finishing a rollout diff --git a/docs/harnesses.md b/docs/harnesses.md index 67ad0f0503..b2ffe536fe 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -76,6 +76,7 @@ Class-level capability flags declare which optional features your harness implem - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). - `LOADS_SESSIONS` (ACP harnesses): indicates that the pinned agent implements ACP `session/load`, so a new process can continue a rollout in a box restored from a snapshot (`agent.checkpoint`). A harness with `SUPPORTS_RESUME` needs nothing more: a restore relaunches it on the transcript. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. +- `SUPPORTS_MEMORY_RESTORE`: indicates that a box restored from a memory snapshot continues the harness (`agent.checkpoint`). The program must run as a job of the box that the host only waits for (`runtime.run(..., program=current_program.get())`), keep no connection to the host open, and reach the host only through the endpoint it was given: the box's relay follows a restored box to its new interception server. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 06ede65093..d98a34584a 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -2,6 +2,8 @@ what a restored attempt must not redo.""" import asyncio +import pickle +import time from types import SimpleNamespace from typing import Literal @@ -11,8 +13,9 @@ from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.configs.client import EvalClientConfig from verifiers.v1.graph import MessageNode +from verifiers.v1.restore import RestorePoint from verifiers.v1.rollout import Rollout, kept -from verifiers.v1.session import RolloutSession +from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import ( Checkpoint, ModelCall, @@ -135,13 +138,22 @@ class SlowBackend: def __init__(self) -> None: self.durable = asyncio.Event() + self.stopped = asyncio.Event() + """Set once a memory snapshot has stopped the box.""" + self.running = asyncio.Event() + """A memory snapshot returns when the test lets the box run again.""" self.fail = False self.taken = 0 self.deleted: list[str] = [] - async def snapshot(self) -> vf.Snapshot: + async def snapshot(self, *, memory: bool = False) -> vf.Snapshot: self.taken += 1 - return vf.Snapshot(id=f"ck{self.taken}") + if memory: + self.stopped.set() + await self.running.wait() + return vf.Snapshot( + id=f"ck{self.taken}", type="memory" if memory else "filesystem" + ) async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: await self.durable.wait() @@ -172,6 +184,9 @@ async def checkpoint_state() -> dict: run._durable = {} run._captures = set() run._closed = False + run._opened = True + run._memory = False + run._capturing = asyncio.Lock() run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] @@ -307,3 +322,152 @@ def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): "a failed rollout retries from the start" ), ] + + +class FakeRelay: + upstream = "https://tunnel.example" + state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} + + def __init__(self) -> None: + self.rerouted: list[str] = [] + + async def reroute(self, upstream: str) -> None: + self.rerouted.append(upstream) + + +def make_memory_rollout( + job: dict | None = {"job_id": "j1"}, +) -> tuple[Rollout, SlowBackend, RolloutSession]: + """A rollout that takes memory snapshots, with its harness segment running `job`.""" + run, backend = make_rollout() + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), trace=run.trace + ) + run._session = session + run._memory = True + run._relay = FakeRelay() # type: ignore[assignment] + run._secret = "box-secret" + run._harness_session = SimpleNamespace( # type: ignore[assignment] + memory_state=lambda: None if job is None else {"job": job} + ) + return run, backend, session + + +async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): + run, backend, session = make_memory_rollout() + session.idempotent_requests = { + "retry:/v1/chat/completions:aa": IdempotentRequest( + binding=("/v1/chat/completions", b"aa"), + response=ReplayResponse(200, b"{}", "application/json"), + completed_at=time.monotonic(), + ), + "old": IdempotentRequest( + binding=("/v1/chat/completions", b"bb"), + response=ReplayResponse(200, b"{}", "application/json"), + completed_at=time.monotonic() - 3600, + ), + "open": IdempotentRequest(binding=("/v1/chat/completions", b"cc")), + } + capture = asyncio.create_task(run._checkpoint(TURN)) + await backend.stopped.wait() + # The box is stopped: nothing it asked for may be answered. + assert not session.flowing.is_set() + assert run._relay.rerouted == [] + backend.running.set() + checkpoint = await capture + assert session.flowing.is_set() + # The stop killed the box's connections: its relay opens new ones. + assert run._relay.rerouted == ["https://tunnel.example"] + assert checkpoint.snapshot == vf.Snapshot(id="ck1", type="memory") + assert checkpoint.node == len(run.trace.nodes) - 1 + assert checkpoint.live == { + "program": {"job": {"job_id": "j1"}}, + "relay": FakeRelay.state, + "secret": "box-secret", + } + # Only a reply the box may not have received yet is kept for its restore. + assert list(checkpoint.replays) == ["retry:/v1/chat/completions:aa"] + assert not checkpoint.pinned + + +async def test_no_memory_snapshot_without_a_running_segment(): + run, backend, session = make_memory_rollout(job=None) + assert await run._checkpoint(TURN) is None + assert backend.taken == 0 and session.flowing.is_set() + + +async def test_a_failed_memory_snapshot_lets_replies_flow_again(): + run, backend, session = make_memory_rollout() + + async def failing(*, memory: bool = False): + raise vf.SandboxError("pause refused") + + backend.snapshot = failing # type: ignore[method-assign] + assert await run._checkpoint(TURN) is None + assert session.flowing.is_set() and run.trace.checkpoints == [] + + +async def test_save_returns_a_restore_point_another_process_can_open(): + run, backend, session = make_memory_rollout() + session.pinned_tool_results = { + ("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned") + } + run.trace.info = {"nonce": "n"} + backend.running.set() + backend.durable.set() + point = await run.save() + assert isinstance(point, RestorePoint) + (checkpoint,) = run.trace.checkpoints + assert checkpoint.pinned and checkpoint.time.end + + point = pickle.loads(pickle.dumps(point)) + assert point.id == run.trace.id + assert point.snapshot == {"id": "ck1", "type": "memory"} + task = vf.Task(vf.TaskData(idx=0, prompt="q")) + trace, restored = point.open(task) + assert trace.id == run.trace.id and trace.num_turns == run.trace.num_turns + assert restored is trace.checkpoints[0] and restored.pinned + assert restored.info == {"nonce": "n"} + assert restored.pins == session.pinned_tool_results + assert restored.live == checkpoint.live + # The restored trace rewinds to the point like a retry's. + assert trace.rewind(restored) == [] + + +async def test_save_waits_for_a_durable_snapshot(): + run, backend, _ = make_memory_rollout() + backend.running.set() + backend.fail = True + backend.durable.set() + assert await run.save() is None + assert run.trace.checkpoints == [] + + +async def test_a_rollout_without_memory_snapshots_saves_nothing(): + run, backend = make_rollout() + assert await run.save() is None + assert backend.taken == 0 + + +async def test_an_owners_snapshot_outlives_the_rollout(): + """A pinned snapshot is not under the keep policy, is not deleted with the run's + own, and becomes durable even when the run settles without a retry to wait for.""" + run, backend, _ = make_memory_rollout() + backend.running.set() + saving = asyncio.create_task(run.save()) + await asyncio.sleep(0) + while not run.trace.checkpoints: + await asyncio.sleep(0) + settling = asyncio.create_task(run._settle_checkpoints(wait=False)) + await asyncio.sleep(0) + assert not settling.done() + backend.durable.set() + await settling + assert await saving is not None + (pinned,) = run.trace.checkpoints + # A newer snapshot of the run's own does not prune the owner's. + newest = await run._checkpoint(TURN) + await run._settle_checkpoints(wait=True) + assert backend.deleted == [] + assert kept(run.trace.checkpoints, "last") == {newest.id} + assert not pinned.deleted From 221b54d1b98d50b1989586c6e797ea38dab29a74 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:40:51 +0000 Subject: [PATCH 27/53] fix lint --- tests/integration/test_memory_checkpoint.py | 17 +++++++++++------ tests/unit/test_checkpoint.py | 12 +++++++----- verifiers/v1/acp/runner.py | 3 ++- verifiers/v1/rollout.py | 2 +- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py index 4368b42d4b..33e4e4bff5 100644 --- a/tests/integration/test_memory_checkpoint.py +++ b/tests/integration/test_memory_checkpoint.py @@ -21,8 +21,8 @@ from verifiers.v1.harness import RESTORE_PROMPT from verifiers.v1.restore import save_all from verifiers.v1.runtimes import PrimeBackend, Snapshot -from verifiers.v1.utils.loaders import harness_class, load_environment from verifiers.v1.types import content_text +from verifiers.v1.utils.loaders import harness_class, load_environment RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} NEEDS = ("memory",) @@ -44,16 +44,21 @@ async def recording(self, **kwargs): return snapshots +async def platform_state(client: AsyncSandboxClient, snapshot_id: str) -> str | None: + """The snapshot's state on prime, or None once it is gone.""" + try: + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None + + async def assert_no_leaks(taken: list[tuple[str, str]]) -> None: """Every snapshot the run took is deleted on the platform.""" leaked = {} async with AsyncSandboxClient() as client: for snapshot_id, _ in taken: - try: - state = (await client.get_checkpoint(snapshot_id)).state - except Exception: # noqa: BLE001 - a deleted snapshot is not found - continue - if state != "DELETING": + state = await platform_state(client, snapshot_id) + if state not in (None, "DELETING"): leaked[snapshot_id] = state assert leaked == {} diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index d98a34584a..c83921fdbb 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -326,9 +326,9 @@ def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): class FakeRelay: upstream = "https://tunnel.example" - state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} def __init__(self) -> None: + self.state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} self.rerouted: list[str] = [] async def reroute(self, upstream: str) -> None: @@ -336,9 +336,11 @@ async def reroute(self, upstream: str) -> None: def make_memory_rollout( - job: dict | None = {"job_id": "j1"}, + running: bool = True, ) -> tuple[Rollout, SlowBackend, RolloutSession]: - """A rollout that takes memory snapshots, with its harness segment running `job`.""" + """A rollout that takes memory snapshots, with or without a harness segment that + runs.""" + job = {"job_id": "j1"} if running else None run, backend = make_rollout() session = RolloutSession( ctx=ModelContext(model="m", client=EvalClientConfig()), trace=run.trace @@ -382,7 +384,7 @@ async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): assert checkpoint.node == len(run.trace.nodes) - 1 assert checkpoint.live == { "program": {"job": {"job_id": "j1"}}, - "relay": FakeRelay.state, + "relay": run._relay.state, "secret": "box-secret", } # Only a reply the box may not have received yet is kept for its restore. @@ -391,7 +393,7 @@ async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): async def test_no_memory_snapshot_without_a_running_segment(): - run, backend, session = make_memory_rollout(job=None) + run, backend, session = make_memory_rollout(running=False) assert await run._checkpoint(TURN) is None assert backend.taken == 0 and session.flowing.is_set() diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index 57b89957d3..0fc1830b40 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -494,7 +494,8 @@ async def connected( with suppress(FileNotFoundError): os.unlink(path) listener = await asyncio.start_unix_server(connected, path, limit=MAX_PACKET_BYTES) - with open(f"{path}.pid", "w") as file: + # A few bytes, once, before any request is served. + with open(f"{path}.pid", "w") as file: # noqa: ASYNC230 file.write(f"{os.getpid()}\n") try: async with listener: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 9c7ea2b811..8c7392dfd3 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -37,6 +37,7 @@ from verifiers.v1.interception import Interception, serve_interception from verifiers.v1.interception.relay import Relay from verifiers.v1.mcp import SharedToolServer, serve_tools +from verifiers.v1.restore import LIVE, RestorePoint from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -45,7 +46,6 @@ make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms -from verifiers.v1.restore import LIVE, RestorePoint from verifiers.v1.session import ( IdempotentRequest, RolloutLimits, From 6dd57f27aaf39321df4114ce94f9b756a86395ed Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 02:49:31 +0000 Subject: [PATCH 28/53] let an owner save rollouts whatever snapshot their turns take Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 20 ++++++++++++++++++++ verifiers/v1/agent.py | 6 +----- verifiers/v1/configs/agent.py | 7 ++++--- verifiers/v1/rollout.py | 13 +++++++++---- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index c83921fdbb..0c127c6fad 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -137,6 +137,7 @@ class SlowBackend: """Captures at once; becomes durable when the test says so.""" def __init__(self) -> None: + self.SUPPORTS_FILESYSTEM_SNAPSHOTS = True self.durable = asyncio.Event() self.stopped = asyncio.Event() """Set once a memory snapshot has stopped the box.""" @@ -174,6 +175,7 @@ def make_rollout( run = Rollout.__new__(Rollout) run.trace = make_trace() # one turn so far run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] + run.harness = SimpleNamespace(can_restore=lambda: True) # type: ignore[assignment] async def checkpoint_state() -> dict: return {} @@ -473,3 +475,21 @@ async def test_an_owners_snapshot_outlives_the_rollout(): assert backend.deleted == [] assert kept(run.trace.checkpoints, "last") == {newest.id} assert not pinned.deleted + + +async def test_filesystem_snapshots_at_turns_leave_saves_to_the_owner(): + """`snapshot = "filesystem"` picks the snapshot of the checkpoints at model turns; + an owner's save still takes a memory snapshot.""" + run, backend, _ = make_memory_rollout() + run._checkpointing = RolloutCheckpointConfig(interval=1, snapshot="filesystem") + + async def checkpoint_state() -> dict: + return {} + + run._harness_session.checkpoint_state = checkpoint_state + backend.running.set() + backend.durable.set() + at_turn = await run._checkpoint(TURN) + assert at_turn.snapshot.type == "filesystem" + point = await run.save() + assert point.snapshot["type"] == "memory" diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 8c3d2e59a3..39fa441d6d 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -666,11 +666,7 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: backend = backend_cls(self.config.runtime) if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: return config - if ( - config.snapshot == "memory" - and harness.SUPPORTS_MEMORY_RESTORE - and backend.SUPPORTS_MEMORY_SNAPSHOTS - ): + if harness.SUPPORTS_MEMORY_RESTORE and backend.SUPPORTS_MEMORY_SNAPSHOTS: return config key = (harness.config.id, self.config.runtime.type) if key not in _checkpointing_warned: diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index d55c6c901a..e7815c020e 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -52,9 +52,10 @@ class RolloutCheckpointConfig(BaseConfig): snapshot is deleted once the run is final.""" snapshot: Literal["memory", "filesystem"] = "memory" - """The snapshot to take: `memory` where the rollout supports it and a filesystem - snapshot elsewhere, or always `filesystem`. A memory snapshot stops the box for a - few seconds; a filesystem snapshot does not.""" + """The snapshot to take at model turns: `memory` where the rollout supports it and + a filesystem snapshot elsewhere, or always `filesystem`. A memory snapshot stops + the box for a few seconds; a filesystem snapshot does not. An owner's save + (`save_all`) takes a memory snapshot either way.""" interval: PositiveInt = 10 """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 8c7392dfd3..dc766a73d5 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -266,8 +266,9 @@ def __init__( gates_tools=harness.SUPPORTS_TOOL_INTERCEPTION, ) self._memory = False - """Whether this run takes memory snapshots: decided at `open`, once the box, - its relay and the task's tool servers are known.""" + """Whether this run can take memory snapshots: decided at `open`, once the box, + its relay and the task's tool servers are known. Its checkpoints at model turns + take them when the config asks for them, and its owner's saves always do.""" self._capturing = asyncio.Lock() if self._restore is not None and self._restore.snapshot.type == "memory": # Set before the box boots: its relay repeats the requests that waited as @@ -498,7 +499,6 @@ async def open(self) -> bool: await self.harness.setup(self._agent_runtime) self._memory = ( self._checkpointing is not None - and self._checkpointing.snapshot == "memory" and self._borrowed_runtime is None and runtime.backend.SUPPORTS_MEMORY_SNAPSHOTS and self.harness.SUPPORTS_MEMORY_RESTORE @@ -946,8 +946,13 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: self.restored_turns is not None and turn_number == self.restored_turns + 1 ): return None - if self._memory: + if self._memory and self._checkpointing.snapshot == "memory": return await self._capture_memory() + if not ( + self.runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS + and self.harness.can_restore() + ): + return None if len(self.trace._pending) > 1: # Another request is in flight (parallel sub-agents share the trace): the # box is not idle, so no snapshot would match this turn's prefix. From 87e35c5329f5ac55e9dd9937e963f18281c2e6c2 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 03:18:04 +0000 Subject: [PATCH 29/53] count a snapshot that is already gone as deleted Co-Authored-By: Claude --- verifiers/v1/runtimes/backends/prime.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index bb84d7a45d..5db29b0598 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -457,6 +457,10 @@ async def discard(cls, snapshots: list[Snapshot]) -> list[str]: try: await client.delete_checkpoint(snapshot.id) except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + if "HTTP 404" in str(e): + # Already gone: an owner that asks again must not ask forever. + deleted.append(snapshot.id) + continue logger.warning( "prime: failed to delete checkpoint %s: %s", snapshot.id, e ) From 562b10c80a016ad6e739e8897e02258d228831ac Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 03:45:35 +0000 Subject: [PATCH 30/53] skip the restore check for a harness that survives the failed reply Co-Authored-By: Claude --- tests/integration/test_memory_checkpoint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py index 33e4e4bff5..202ead9482 100644 --- a/tests/integration/test_memory_checkpoint.py +++ b/tests/integration/test_memory_checkpoint.py @@ -77,6 +77,10 @@ async def restored_trace( ) (trace,) = episode.traces assert trace.ok, [error.message for error in trace.errors] + if not trace.errors: + # The harness handled the failed reply itself and finished its run: nothing + # failed, so nothing was restored. + pytest.skip(f"{harness.id} recovers from a failed model call by itself") # The process `setup` started kept its memory across the restore. assert trace.reward == 1.0 assert [error.type for error in trace.errors] == ["ProviderError"] From d10b76dbebe8252dcc730e1c1e4f84dcee6dc36b Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 04:14:30 +0000 Subject: [PATCH 31/53] say why a running rollout takes no restore point Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 31 +++++++++++++++++++++++- verifiers/v1/restore.py | 40 +++++++++++++++++++++++++++---- verifiers/v1/rollout.py | 44 +++++++++++++++++++++-------------- 3 files changed, 93 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 0c127c6fad..212ce206c2 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -13,7 +13,7 @@ from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.configs.client import EvalClientConfig from verifiers.v1.graph import MessageNode -from verifiers.v1.restore import RestorePoint +from verifiers.v1.restore import RestorePoint, unsaveable from verifiers.v1.rollout import Rollout, kept from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import ( @@ -188,6 +188,8 @@ async def checkpoint_state() -> dict: run._closed = False run._opened = True run._memory = False + run.unsaveable = "the 'fake' runtime takes no memory snapshots" + run.save_skipped = None run._capturing = asyncio.Lock() run._pruning = asyncio.Lock() run.deleted_snapshots = set() @@ -349,6 +351,7 @@ def make_memory_rollout( ) run._session = session run._memory = True + run.unsaveable = None run._relay = FakeRelay() # type: ignore[assignment] run._secret = "box-secret" run._harness_session = SimpleNamespace( # type: ignore[assignment] @@ -451,6 +454,32 @@ async def test_a_rollout_without_memory_snapshots_saves_nothing(): run, backend = make_rollout() assert await run.save() is None assert backend.taken == 0 + assert run.save_skipped == "the 'fake' runtime takes no memory snapshots" + + +async def test_a_save_between_segments_names_its_reason(): + run, _, _ = make_memory_rollout(running=False) + assert await run.save() is None + assert run.save_skipped == "its harness program is not running (setup or scoring)" + + +def test_unsaveable_names_what_rules_memory_snapshots_out(): + def reason(harness: str, runtime: vf.RuntimeConfig, **config) -> str | None: + return unsaveable( + vf.AgentConfig(harness={"id": harness}, runtime=runtime, **config) + ) + + assert reason("bash", vf.PrimeConfig()) is None + assert reason("rlm", vf.PrimeConfig()) is None + assert reason("bash", vf.SubprocessConfig()) == ( + "the 'subprocess' runtime takes no memory snapshots" + ) + assert reason("terminus-2", vf.PrimeConfig()) == ( + "harness 'terminus-2' cannot continue from a memory snapshot" + ) + assert reason("bash", vf.PrimeConfig(), checkpoint=None) == ( + "rollout checkpointing is off (agent.checkpoint)" + ) async def test_an_owners_snapshot_outlives_the_rollout(): diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py index 752767c534..50c92af9a5 100644 --- a/verifiers/v1/restore.py +++ b/verifiers/v1/restore.py @@ -31,6 +31,8 @@ from verifiers.v1.utils.logging import get_logger if TYPE_CHECKING: + from verifiers.v1.configs.agent import AgentConfig + from verifiers.v1.harness import Harness from verifiers.v1.rollout import Rollout from verifiers.v1.task import Task @@ -119,18 +121,48 @@ def open(self, task: Task) -> tuple[Trace, Checkpoint]: return trace, checkpoint -async def save_all() -> list[RestorePoint]: +def unsaveable(config: AgentConfig, harness: Harness | None = None) -> str | None: + """Why the rollouts of an agent with `config` take no memory snapshots, so `save_all` + returns no restore point for them; None when they take them. A task with tool + servers or services of its own rules its rollout out as well, which only the + rollout knows.""" + from verifiers.v1.runtimes import backend_cls + from verifiers.v1.utils.loaders import harness_class + + if config.checkpoint is None: + return "rollout checkpointing is off (agent.checkpoint)" + if not backend_cls(config.runtime).SUPPORTS_MEMORY_SNAPSHOTS: + return f"the {config.runtime.type!r} runtime takes no memory snapshots" + if harness is not None: + supported, name = harness.SUPPORTS_MEMORY_RESTORE, harness.config.id + elif config.harness is not None: + supported = harness_class(config.harness.id).SUPPORTS_MEMORY_RESTORE + name = config.harness.id + else: + return None + if not supported: + return f"harness {name!r} cannot continue from a memory snapshot" + return None + + +async def save_all(skipped: dict[str, int] | None = None) -> list[RestorePoint]: """Take a memory checkpoint of every running rollout of this process and return their restore points. A rollout that takes no memory snapshots, runs no harness segment right now, or whose snapshot fails has none: it starts over when its owner - runs it again. The snapshots are pinned: the caller owns them.""" + runs it again. `skipped` collects why, as a count per reason. The snapshots are + pinned: the caller owns them.""" async def save(rollout: Rollout) -> RestorePoint | None: try: - return await rollout.save() + point = await rollout.save() + reason = rollout.save_skipped except Exception: logger.warning("saving rollout %s failed", rollout.trace.id, exc_info=True) - return None + point, reason = None, "its save raised an error" + if point is None and skipped is not None: + reason = reason or "unknown" + skipped[reason] = skipped.get(reason, 0) + 1 + return point rollouts = list(LIVE) points = [ diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index dc766a73d5..11c131affd 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -37,7 +37,7 @@ from verifiers.v1.interception import Interception, serve_interception from verifiers.v1.interception.relay import Relay from verifiers.v1.mcp import SharedToolServer, serve_tools -from verifiers.v1.restore import LIVE, RestorePoint +from verifiers.v1.restore import LIVE, RestorePoint, unsaveable from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -269,6 +269,11 @@ def __init__( """Whether this run can take memory snapshots: decided at `open`, once the box, its relay and the task's tool servers are known. Its checkpoints at model turns take them when the config asks for them, and its owner's saves always do.""" + self.unsaveable: str | None = "it has not booted its box yet" + """Why this run takes no memory snapshots, so `save` returns nothing; None when + it takes them.""" + self.save_skipped: str | None = None + """Why the last `save` returned nothing.""" self._capturing = asyncio.Lock() if self._restore is not None and self._restore.snapshot.type == "memory": # Set before the box boots: its relay repeats the requests that waited as @@ -497,18 +502,18 @@ async def open(self) -> bool: # A box restored from memory still runs the harness. if not memory_restore: await self.harness.setup(self._agent_runtime) - self._memory = ( - self._checkpointing is not None - and self._borrowed_runtime is None - and runtime.backend.SUPPORTS_MEMORY_SNAPSHOTS - and self.harness.SUPPORTS_MEMORY_RESTORE + self.unsaveable = unsaveable(self.trace.agent.config, self.harness) or ( + "it runs in a borrowed box" + if self._borrowed_runtime is not None # Everything the box reaches the host through must follow it to a # new box: the relay does, tool servers and services do not. - and self._relay is not None - and not toolsets - and not self._shared_tools - and self.deployment.spec is None + else "its task has tool servers or services of its own" + if toolsets + or self._shared_tools + or self.deployment.spec is not None + else None ) + self._memory = self.unsaveable is None and self._relay is not None if memory_restore and not self._memory: raise SandboxError( "the rollout cannot continue from its memory snapshot" @@ -1009,13 +1014,16 @@ async def save(self) -> RestorePoint | None: (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`). None when the rollout takes no memory snapshots (`RolloutCheckpointConfig`), runs no segment, or the snapshot did not become restorable.""" - if not self._opened or self._closed or not self._memory: - logger.debug( - "Not saving rollout %s: memory=%s closed=%s", - self.trace.id, - self._memory, - self._closed, - ) + self.save_skipped = None + if self._closed: + self.save_skipped = "it is finishing" + elif not self._memory: + self.save_skipped = self.unsaveable or "its box has no relay" + elif self._harness_session is None or ( + self._harness_session.memory_state() is None + ): + self.save_skipped = "its harness program is not running (setup or scoring)" + if self.save_skipped is not None: return None task = asyncio.current_task() if task is not None: @@ -1025,6 +1033,7 @@ async def save(self) -> RestorePoint | None: finally: self._captures.discard(task) if checkpoint is None: + self.save_skipped = "its memory snapshot failed" return None # Before the next await: the trace moves on once replies flow again. point = RestorePoint.of(self.trace, checkpoint) @@ -1032,6 +1041,7 @@ async def save(self) -> RestorePoint | None: if durable is not None: await asyncio.wait({durable}) if not checkpoint.time.end: + self.save_skipped = "its memory snapshot did not become restorable" return None return point From d85c50bfe82bbf5e942484e6d83305ab193b5b14 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 04:28:54 +0000 Subject: [PATCH 32/53] return a saved rollout's restore point before its snapshot is durable A save can also hold the rollout at its restore point until the owner releases it, so the owner records which rollouts still run without one of them finishing meanwhile. Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 40 +++++++++++++++++++++++++++---- verifiers/v1/restore.py | 16 ++++++++++--- verifiers/v1/rollout.py | 44 ++++++++++++++++++++++++----------- 3 files changed, 78 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 212ce206c2..3004316098 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -190,6 +190,8 @@ async def checkpoint_state() -> dict: run._memory = False run.unsaveable = "the 'fake' runtime takes no memory snapshots" run.save_skipped = None + run._held = False + run._release_timer = None run._capturing = asyncio.Lock() run._pruning = asyncio.Lock() run.deleted_snapshots = set() @@ -425,7 +427,7 @@ async def test_save_returns_a_restore_point_another_process_can_open(): point = await run.save() assert isinstance(point, RestorePoint) (checkpoint,) = run.trace.checkpoints - assert checkpoint.pinned and checkpoint.time.end + assert checkpoint.pinned point = pickle.loads(pickle.dumps(point)) assert point.id == run.trace.id @@ -441,13 +443,41 @@ async def test_save_returns_a_restore_point_another_process_can_open(): assert trace.rewind(restored) == [] -async def test_save_waits_for_a_durable_snapshot(): +async def test_save_returns_before_the_snapshot_is_durable(): + """The owner gets the restore point once the box runs again; the snapshot becomes + durable in the background, and the rollout's teardown waits for it.""" run, backend, _ = make_memory_rollout() backend.running.set() - backend.fail = True + point = await run.save() + (checkpoint,) = run.trace.checkpoints + assert point is not None and not checkpoint.time.end backend.durable.set() - assert await run.save() is None - assert run.trace.checkpoints == [] + await run._settle_checkpoints(wait=False) + assert checkpoint.time.end + + +async def test_a_held_save_answers_the_box_only_after_release(): + run, backend, session = make_memory_rollout() + backend.running.set() + assert await run.save(hold=True) is not None + assert not session.flowing.is_set() + # A checkpoint of the run's own under the hold does not end it. + await run._checkpoint(TURN) + assert not session.flowing.is_set() + run.release() + assert session.flowing.is_set() and run._release_timer is None + + +async def test_a_failed_held_save_does_not_hold_the_box(): + run, backend, session = make_memory_rollout() + + async def failing(*, memory: bool = False): + raise vf.SandboxError("pause refused") + + backend.snapshot = failing # type: ignore[method-assign] + assert await run.save(hold=True) is None + assert session.flowing.is_set() + assert run.save_skipped == "its memory snapshot failed" async def test_a_rollout_without_memory_snapshots_saves_nothing(): diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py index 50c92af9a5..41fab1a30f 100644 --- a/verifiers/v1/restore.py +++ b/verifiers/v1/restore.py @@ -145,16 +145,26 @@ def unsaveable(config: AgentConfig, harness: Harness | None = None) -> str | Non return None -async def save_all(skipped: dict[str, int] | None = None) -> list[RestorePoint]: +def release_all() -> None: + """Let every rollout that `save_all(hold=True)` holds go on.""" + for rollout in list(LIVE): + rollout.release() + + +async def save_all( + skipped: dict[str, int] | None = None, *, hold: bool = False +) -> list[RestorePoint]: """Take a memory checkpoint of every running rollout of this process and return their restore points. A rollout that takes no memory snapshots, runs no harness segment right now, or whose snapshot fails has none: it starts over when its owner runs it again. `skipped` collects why, as a count per reason. The snapshots are - pinned: the caller owns them.""" + pinned: the caller owns them. With `hold`, each saved rollout waits at its restore + point until `release_all`, so the caller can record which rollouts are still + running without one of them finishing meanwhile.""" async def save(rollout: Rollout) -> RestorePoint | None: try: - point = await rollout.save() + point = await rollout.save(hold=hold) reason = rollout.save_skipped except Exception: logger.warning("saving rollout %s failed", rollout.trace.id, exc_info=True) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 11c131affd..38f5b0c9c1 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -87,6 +87,8 @@ class RolloutTimeouts: isolated verifier setup and retries. Finalization has its own budget.""" +HOLD_SECONDS = 120 +"""How long a saved rollout waits for its owner's `release` before it goes on by itself.""" REPLAY_SECONDS = 30 """A memory checkpoint keeps the replies the server sent this recently: one of them may not have reached the box when the snapshot stopped it.""" @@ -274,6 +276,9 @@ def __init__( it takes them.""" self.save_skipped: str | None = None """Why the last `save` returned nothing.""" + self._held = False + """Whether an owner holds the box's replies past its save (`save(hold=True)`).""" + self._release_timer: asyncio.TimerHandle | None = None self._capturing = asyncio.Lock() if self._restore is not None and self._restore.snapshot.type == "memory": # Set before the box boots: its relay repeats the requests that waited as @@ -1008,12 +1013,15 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: ) return checkpoint - async def save(self) -> RestorePoint | None: + async def save(self, *, hold: bool = False) -> RestorePoint | None: """Take a memory checkpoint of the running rollout now and return its restore point, for an owner that continues the rollout in another process - (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`). None when - the rollout takes no memory snapshots (`RolloutCheckpointConfig`), runs no - segment, or the snapshot did not become restorable.""" + (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`); it becomes + restorable from any node in the background, and this run's teardown waits for + that. With `hold`, nothing the box asked for is answered until `release`, so + the rollout cannot move past its restore point while the owner records it. + None when the rollout takes no memory snapshots (`unsaveable`), runs no + segment, or the snapshot failed; `save_skipped` then says which.""" self.save_skipped = None if self._closed: self.save_skipped = "it is finishing" @@ -1028,22 +1036,29 @@ async def save(self) -> RestorePoint | None: task = asyncio.current_task() if task is not None: self._captures.add(task) + self._held = hold try: checkpoint = await self._capture_memory(pinned=True) finally: self._captures.discard(task) if checkpoint is None: + self.release() self.save_skipped = "its memory snapshot failed" return None - # Before the next await: the trace moves on once replies flow again. - point = RestorePoint.of(self.trace, checkpoint) - durable = self._durable.get(checkpoint.id) - if durable is not None: - await asyncio.wait({durable}) - if not checkpoint.time.end: - self.save_skipped = "its memory snapshot did not become restorable" - return None - return point + if hold: + # An owner that never releases must not stall the rollout for good. + self._release_timer = asyncio.get_running_loop().call_later( + HOLD_SECONDS, self.release + ) + return RestorePoint.of(self.trace, checkpoint) + + def release(self) -> None: + """Answer the box again after a `save(hold=True)`.""" + self._held = False + if self._release_timer is not None: + self._release_timer.cancel() + self._release_timer = None + self._session.resume() async def _capture_memory(self, *, pinned: bool = False) -> Checkpoint | None: """Snapshot the box with its memory. Nothing the box asked for is answered @@ -1096,7 +1111,8 @@ async def _capture_memory(self, *, pinned: bool = False) -> Checkpoint | None: exc_info=True, ) finally: - session.resume() + if not self._held: + session.resume() checkpoint = Checkpoint( id=snapshot.id, node=node, From bd383bb8efd535181fc7152de4691eff993fe416 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 04:46:30 +0000 Subject: [PATCH 33/53] ask again when a box does not resume after its memory snapshot A pause frees the box's memory on its node. The platform refuses the resume while another box holds that room, and a box left paused never runs its rollout again. Co-Authored-By: Claude --- verifiers/v1/runtimes/backends/prime.py | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 5db29b0598..3093f532cb 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -60,6 +60,7 @@ """How long a pause or a resume gets to take effect; they take seconds.""" _DEAD_STATUSES = frozenset({"ERROR", "TERMINATED", "TIMEOUT"}) _STATUS_POLL_SECONDS = 0.3 +_RESUME_RETRY_SECONDS = 600 """How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" @@ -387,8 +388,7 @@ async def create() -> Snapshot: except BaseException: self.stray_snapshots.remove(snapshot) raise - await self._client.resume(self.info.id) - await self._settle("RUNNING", "RESUME_FAILED") + await self._resume() finally: self._awake.set() return snapshot @@ -402,6 +402,32 @@ async def create() -> Snapshot: except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e + async def _resume(self) -> None: + """Resume the paused box. The pause freed its memory on its node, which another + box may have taken: the platform then refuses the resume until there is room + again, so ask until it runs or `_RESUME_RETRY_SECONDS` passed.""" + deadline = time.monotonic() + _RESUME_RETRY_SECONDS + delay = 1.0 + while True: + try: + await self._client.resume(self.info.id) + await self._settle("RUNNING", "RESUME_FAILED") + return + except Exception as e: + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not resume after its " + f"memory snapshot: {e}" + ) from e + logger.warning( + "prime: sandbox %s did not resume, asking again in %.0fs: %s", + self.info.id, + delay, + e, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 15) + def _stopped_by_snapshot(self, error: Exception) -> bool: """Whether a call failed because a memory snapshot had stopped the box: the call is safe to repeat once the box runs again.""" From a1ca7b9a9d1887df0a40a05f315fe3cc28e3b084 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 04:46:43 +0000 Subject: [PATCH 34/53] check the box before asking it to resume again --- verifiers/v1/runtimes/backends/prime.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 3093f532cb..5bac6d5a1e 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -410,6 +410,9 @@ async def _resume(self) -> None: delay = 1.0 while True: try: + # An earlier request may have taken effect after it was given up on. + if (await self._client.get(self.info.id)).status == "RUNNING": + return await self._client.resume(self.info.id) await self._settle("RUNNING", "RESUME_FAILED") return From 0c8eab643401d36090edae664a85e863fb362861 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 07:33:17 +0000 Subject: [PATCH 35/53] start over when a restore point leads nowhere A box created from a memory snapshot continues the paused box's idle clock. In a training run resumed 2 h 41 min after its checkpoint, the platform ended 7 of 46 such boxes as idle before they ran, and each episode failed without a retry. Create the box again up to three times when it ends before it runs. If the restore point of an owner still leads nowhere, start the rollout over without spending one of its retries: that is what it would have done without the point. Co-Authored-By: Claude --- tests/integration/test_memory_checkpoint.py | 56 +++++++++ verifiers/v1/agent.py | 19 ++++ verifiers/v1/runtimes/backends/prime.py | 120 ++++++++++++-------- 3 files changed, 148 insertions(+), 47 deletions(-) diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py index 202ead9482..d705965ccc 100644 --- a/tests/integration/test_memory_checkpoint.py +++ b/tests/integration/test_memory_checkpoint.py @@ -213,3 +213,59 @@ async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_r snapshot_id ] await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True +) +async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( + taken, harness, harness_runtime +): + """A restore point whose snapshot the platform no longer restores costs the rollout + no retry: it starts over under the same trace id and finishes.""" + config = _eval_config( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": 99}}, + max_turns=12, + max_tokens=4096, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + first = load_environment(config.env) + (task,) = list(first.taskset) + async with first.serving(): + (slot,) = first.slots(task) + running = asyncio.ensure_future(first.run_slot(slot, ctx)) + try: + async with asyncio.timeout(600): + while not (slot.traces and slot.traces[0].num_turns >= 1): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + snapshot_id = point.snapshot["id"] + point.snapshot["id"] = "no-such-checkpoint" + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point.id + assert trace.reward == 1.0 + assert trace.attempt == 3 + + async with AsyncSandboxClient() as client, asyncio.timeout(300): + while (await client.get_checkpoint(snapshot_id)).state != "DURABLE": + await asyncio.sleep(2) + assert await PrimeBackend.discard([Snapshot(id=snapshot_id, type="memory")]) == [ + snapshot_id + ] + await assert_no_leaks(taken) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 39fa441d6d..d8129efd62 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -817,6 +817,25 @@ async def run( failed_restores.add(target.id) if trace.ok: break + if ( + restore is not None + and target is not None + and target.id == restore.checkpoint + and target.id in failed_restores + ): + # The owner's restore point is an offer, not an attempt of + # this run: without it the rollout would have started over, + # so it does that now and keeps its retries. + logger.warning( + "Starting rollout over: restore point %s led nowhere - %s", + target.id, + clip(trace.errors[-1].message) if trace.errors else "", + ) + history.extend(trace.errors) + target = None + attempt += 1 + budget += 1 + continue if attempt >= budget: break if runtime is not None: diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 5bac6d5a1e..606214333e 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -60,6 +60,8 @@ """How long a pause or a resume gets to take effect; they take seconds.""" _DEAD_STATUSES = frozenset({"ERROR", "TERMINATED", "TIMEOUT"}) _STATUS_POLL_SECONDS = 0.3 +_FORK_ATTEMPTS = 3 +"""Creates of a box from a memory snapshot before its restore counts as failed.""" _RESUME_RETRY_SECONDS = 600 """How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it after the request returns, and only a durable one restores.""" @@ -248,7 +250,11 @@ def bind(self, *, env: Mapping[str, str] | None = None) -> Runtime: return super().bind(env=env).with_user(self.user) async def start(self) -> None: - from prime_sandboxes import APIError, CreateSandboxRequest + from prime_sandboxes import ( + APIError, + CreateSandboxRequest, + SandboxNotRunningError, + ) started = time.perf_counter() self._client = _lease_client() @@ -294,54 +300,74 @@ async def start(self) -> None: scope = run_scope() labels = [*BASE_LABELS, *self.config.labels, scope] try: - async with ( - creation_limiter( - (self.config.creates_per_min or 0) / 60, - "prime-sandbox", - scope, - ) - or contextlib.nullcontext() - ): - for name in dict.fromkeys(m.name for m in self.config.mounts.values()): - try: - await self._client.volumes.create(name) - except APIError as error: - # Create returns 409 when this scoped name already exists, - # including when another runtime creates it concurrently. - if not str(error).startswith("HTTP 409:"): - raise - - # Shielded through the id capture: a cancel that aborts the POST - # mid-flight leaves the platform creating a sandbox this side - # never learned the id of — teardown() then cannot delete it - async def create_and_capture_id(): - sandbox = await self._client.create( - CreateSandboxRequest( - name=self.name, - labels=list(dict.fromkeys(labels)), - **({} if memory else {"environment_vars": self.env}), - **source, - **{k: v for k, v in options.items() if v is not None}, + attempts = _FORK_ATTEMPTS if memory else 1 + for attempt in range(attempts): + async with ( + creation_limiter( + (self.config.creates_per_min or 0) / 60, + "prime-sandbox", + scope, + ) + or contextlib.nullcontext() + ): + for name in dict.fromkeys( + m.name for m in self.config.mounts.values() + ): + try: + await self._client.volumes.create(name) + except APIError as error: + # Create returns 409 when this scoped name already exists, + # including when another runtime creates it concurrently. + if not str(error).startswith("HTTP 409:"): + raise + + # Shielded through the id capture: a cancel that aborts the POST + # mid-flight leaves the platform creating a sandbox this side + # never learned the id of — teardown() then cannot delete it + async def create_and_capture_id(): + sandbox = await self._client.create( + CreateSandboxRequest( + name=self.name, + labels=list(dict.fromkeys(labels)), + **({} if memory else {"environment_vars": self.env}), + **source, + **{k: v for k, v in options.items() if v is not None}, + ) ) + self.info.id = sandbox.id + return sandbox + + sandbox = await run_shielded(create_and_capture_id()) + # The create response says whether the platform already has the image: + # `pending_image_build_id` set means a first-use auto-build is running and the + # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase + # its own budget, separate from the normal boot attempts). + self.info.image_cached = sandbox.pending_image_build_id is None + if not self.info.image_cached: + logger.warning( + "Building image %s on the platform for prime runtime %s - the " + "first use of an image can take ~10 minutes, later runs start " + "in seconds", + self.config.image, + self.info.id, + ) + try: + await self._client.wait_for_creation(self.info.id, max_attempts=900) + break + except SandboxNotRunningError as error: + # A box that continues an old memory snapshot also continues its + # idle clock: the platform can end it as idle before it runs. + if attempt + 1 == attempts: + raise + logger.warning( + "Prime runtime %s ended before it ran from memory snapshot " + "%s; creating it again (attempt %d/%d) - %s", + self.info.id, + self.config.checkpoint, + attempt + 2, + attempts, + error, ) - self.info.id = sandbox.id - return sandbox - - sandbox = await run_shielded(create_and_capture_id()) - # The create response says whether the platform already has the image: - # `pending_image_build_id` set means a first-use auto-build is running and the - # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase - # its own budget, separate from the normal boot attempts). - self.info.image_cached = sandbox.pending_image_build_id is None - if not self.info.image_cached: - logger.warning( - "Building image %s on the platform for prime runtime %s - the " - "first use of an image can take ~10 minutes, later runs start " - "in seconds", - self.config.image, - self.info.id, - ) - await self._client.wait_for_creation(self.info.id, max_attempts=900) logger.debug( "Started prime runtime %s in %s image=%s%s", self.info.id, From 569389fa38772a35f3413727739868e4301ccf51 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 08:08:02 +0000 Subject: [PATCH 36/53] take memory snapshots only A memory snapshot continues every process of the box, so it does all that a filesystem snapshot with a new harness process did, and needs nothing from the harness. Remove the filesystem path: the snapshot type and its config field, the restore prompt, session loading in the ACP runner, and the harness state paths it needed. A rollout that cannot take memory snapshots (a runtime without them, a harness that does not run detached, tool servers or services of its own) takes no checkpoints, and a retry starts over. Co-Authored-By: Claude --- .github/workflows/tests.yml | 2 - docs/agent.md | 17 +- docs/harnesses.md | 1 - .../fixtures/checkpoint_restore_v1.py | 133 -------- .../integration/fixtures/memory_restore_v1.py | 4 +- tests/integration/fixtures/pyproject.toml | 1 - tests/integration/grid.py | 9 +- tests/integration/test_checkpoint.py | 284 +++++++++++------- tests/integration/test_memory_checkpoint.py | 271 ----------------- tests/unit/test_checkpoint.py | 226 +++++--------- verifiers/v1/acp/__init__.py | 44 +-- verifiers/v1/acp/runner.py | 44 +-- verifiers/v1/agent.py | 22 +- verifiers/v1/configs/agent.py | 22 +- verifiers/v1/harness.py | 35 +-- verifiers/v1/harnesses/claude_code/harness.py | 9 +- verifiers/v1/harnesses/codex/harness.py | 4 +- verifiers/v1/harnesses/rlm/harness.py | 18 +- verifiers/v1/interception/server.py | 2 +- verifiers/v1/rollout.py | 142 ++------- verifiers/v1/runtimes/backends/prime.py | 48 ++- verifiers/v1/runtimes/base.py | 36 +-- verifiers/v1/session.py | 23 +- verifiers/v1/trace.py | 27 +- 24 files changed, 377 insertions(+), 1047 deletions(-) delete mode 100644 tests/integration/fixtures/checkpoint_restore_v1.py delete mode 100644 tests/integration/test_memory_checkpoint.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f3caea2975..19409ffe18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -99,8 +99,6 @@ jobs: tests: tests/integration/test_resume_tool.py - behavior: Checkpoint Restore tests: tests/integration/test_checkpoint.py - - behavior: Memory Checkpoint Restore - tests: tests/integration/test_memory_checkpoint.py - behavior: User Sim tests: tests/integration/test_user_sim.py - behavior: Agent User diff --git a/docs/agent.md b/docs/agent.md index 24c0730d20..dac83f6c22 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,18 +77,23 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. Each checkpoint records the type of its snapshot: +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. -- A **memory** snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. It needs a runtime that takes memory snapshots (prime), a harness that runs detached from the host (`SUPPORTS_MEMORY_RESTORE`: null, bash and the ACP harnesses), and a rollout without tool servers or services of its own. -- A **filesystem** snapshot holds the box's files. The retry starts a new harness process in the restored box and tells it that its processes are gone. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). +A snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. -A rollout takes memory snapshots where it can and filesystem snapshots elsewhere. Without either, a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +A rollout takes snapshots only when all of these hold. Otherwise a retry starts over, with a warning once per harness and runtime: + +- The runtime takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`: prime). +- The harness runs detached from the host (`SUPPORTS_MEMORY_RESTORE`: null, bash and the ACP harnesses). +- The rollout runs in a box of its own, not in a borrowed one. +- The rollout has no tool servers, no shared tools and no services of its own. They reach the host at an address that does not follow the box to its restore. + +A retry also starts over when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] -snapshot = "memory" # or "filesystem": never stop the box for a snapshot interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` -An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns a `RestorePoint` for each: plain data the owner stores. A later process continues a rollout from its point with `RunSlot(task, id=..., restore=point)` (or `Agent.run(task, restore=point)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. +An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns a `RestorePoint` for each: plain data the owner stores. A later process continues a rollout from its point with `RunSlot(task, id=..., restore=point)` (or `Agent.run(task, restore=point)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. A restore point continues one rollout: an episode with more than one agent run cannot continue from the point of one of them, and starts over. diff --git a/docs/harnesses.md b/docs/harnesses.md index b2ffe536fe..383e69c6ee 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,6 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `LOADS_SESSIONS` (ACP harnesses): indicates that the pinned agent implements ACP `session/load`, so a new process can continue a rollout in a box restored from a snapshot (`agent.checkpoint`). A harness with `SUPPORTS_RESUME` needs nothing more: a restore relaunches it on the transcript. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_MEMORY_RESTORE`: indicates that a box restored from a memory snapshot continues the harness (`agent.checkpoint`). The program must run as a job of the box that the host only waits for (`runtime.run(..., program=current_program.get())`), keep no connection to the host open, and reach the host only through the endpoint it was given: the box's relay follows a restored box to its new interception server. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/integration/fixtures/checkpoint_restore_v1.py b/tests/integration/fixtures/checkpoint_restore_v1.py deleted file mode 100644 index addfaf2b2e..0000000000 --- a/tests/integration/fixtures/checkpoint_restore_v1.py +++ /dev/null @@ -1,133 +0,0 @@ -"""checkpoint-restore: an agentic task whose first attempt dies mid-rollout, so the retry -restores the box's newest snapshot instead of starting over. - -`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model -for two tool calls that build on each other (the second needs the first's token, so they -take two turns), and the reward reads the second one's file back out of the box. The -tools are colocated: they run in the box, so their files are in its snapshots, and any -harness with MCP tools can drive them. A request `@intercept` raises a `ProviderError` -once the first attempt has made two turns — after the snapshot of turn 2 exists. The -restored attempt continues from that snapshot in a new harness process: it never sees -the injected failure again (it fires once per task), finds the earlier calls' files, and -finishes. Needs a harness and a runtime that both support the snapshot type -(`SUPPORTS__SNAPSHOTS`; today prime's filesystem snapshots). -""" - -import shutil -import uuid -from pathlib import Path - -import verifiers.v1 as vf -from verifiers.v1.errors import SandboxError -from verifiers.v1.runtimes import Runtime - -MARKER = "marker.txt" -STEP_ONE = "step1.txt" -TOKEN = "token.txt" -DONE = "done.txt" -FAIL_AFTER_TURNS = 2 - - -class RestoreState(vf.State): - workdir: str = "" - """The box's working directory, where `setup` put the marker.""" - - -class StepsToolset(vf.Toolset[vf.ToolsetConfig, RestoreState]): - TOOL_PREFIX = "steps" - - @vf.tool - def copy_marker(self) -> str: - """Copy the marker file to the step-one file. Returns the token `write_done` - needs.""" - workdir = Path(self.state.workdir) - shutil.copyfile(workdir / MARKER, workdir / STEP_ONE) - token = uuid.uuid4().hex[:8] - (workdir / TOKEN).write_text(token) - return f"copied; token={token}" - - @vf.tool - def write_done(self, token: str) -> str: - """Write the step-one file's content to the done file. Needs the token that - `copy_marker` returned.""" - workdir = Path(self.state.workdir) - if not (workdir / TOKEN).is_file() or token != (workdir / TOKEN).read_text(): - return "wrong token: call copy_marker first and pass the token it returns" - shutil.copyfile(workdir / STEP_ONE, workdir / DONE) - return "written" - - -class CheckpointRestoreTaskConfig(vf.TaskConfig): - tools: vf.ToolsetConfig = vf.ToolsetConfig(colocated=True) - - -class CheckpointRestoreConfig(vf.TasksetConfig): - task: CheckpointRestoreTaskConfig = CheckpointRestoreTaskConfig() - - -class CheckpointRestoreTask( - vf.Task[vf.TaskData, RestoreState, CheckpointRestoreTaskConfig] -): - failed = False - """Whether the injected failure fired: once per task, so a retry from scratch - (checkpoints off) also gets through.""" - - @classmethod - def toolsets(cls, config: CheckpointRestoreTaskConfig) -> list[vf.Toolset]: - return [StepsToolset(config.tools)] - - async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: - nonce = uuid.uuid4().hex - trace.info["nonce"] = nonce - await runtime.write(MARKER, nonce.encode()) - result = await runtime.run(["pwd"], {}) - trace.state.workdir = result.stdout.strip() - - @vf.intercept - def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: - # Request hooks also judge each proposed tool call after a turn commits, over - # the branch plus an empty probe result; fail only as a real turn goes upstream - # (every tool returns text, so a real result is never empty). - last = request.messages[-1] - probe = isinstance(last, vf.ToolMessage) and not last.content - if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not self.failed: - self.failed = True - # A 4xx: harness SDKs end the turn at once instead of retrying it until the - # rollout's deadline. - raise vf.ProviderError("injected mid-rollout failure", status_code=400) - - @vf.reward(weight=1.0) - async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: - try: - content = (await runtime.read(DONE)).decode(errors="replace") - except (SandboxError, OSError, ValueError): - return 0.0 - return float(content.strip() == trace.info["nonce"]) - - -class CheckpointRestoreTaskset( - vf.Taskset[CheckpointRestoreTask, CheckpointRestoreConfig] -): - def load(self) -> list[CheckpointRestoreTask]: - return [ - CheckpointRestoreTask( - vf.TaskData( - idx=0, - prompt=( - "Use the tools of the `steps` MCP server. Make one tool call " - "per turn and wait for each result before the next.\n" - "1. Call `copy_marker`.\n" - "2. Call `write_done` with the token `copy_marker` returned.\n" - "3. Reply with the single word: done." - ), - ), - self.config.task, - ) - ] - - -__all__ = ["CheckpointRestoreTaskset"] - - -if __name__ == "__main__": - StepsToolset.run() diff --git a/tests/integration/fixtures/memory_restore_v1.py b/tests/integration/fixtures/memory_restore_v1.py index 87884d5dfe..69149b1033 100644 --- a/tests/integration/fixtures/memory_restore_v1.py +++ b/tests/integration/fixtures/memory_restore_v1.py @@ -2,8 +2,8 @@ box's newest memory snapshot and continues the harness program the box still runs. `setup` starts a process in the box that holds a nonce only in its memory and hands it -to whoever reads `nonce.fifo`. The nonce is on no disk, so a box that boots from a -filesystem snapshot cannot answer; a box that continues from a memory snapshot can. +to whoever reads `nonce.fifo`. The nonce is on no disk, so only a box that continues +the process from a memory snapshot can answer. The reward reads the nonce from the process. The task asks a harness with a shell for two commands that build on each other, and a metric checks the file they leave. diff --git a/tests/integration/fixtures/pyproject.toml b/tests/integration/fixtures/pyproject.toml index 675c238d8b..917d9ba656 100644 --- a/tests/integration/fixtures/pyproject.toml +++ b/tests/integration/fixtures/pyproject.toml @@ -15,7 +15,6 @@ build-backend = "hatchling.build" [tool.hatch.build] include = [ "agent_user.py", - "checkpoint_restore_v1.py", "counter_tool.py", "echo_acp_resume.py", "echo_tool.py", diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 5ec8c84a0e..a9475fcab3 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,9 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). A `snapshots` need asks the runtime for -`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`; a `memory` -need asks them for `SUPPORTS_MEMORY_SNAPSHOTS` and `Harness.SUPPORTS_MEMORY_RESTORE`. +`SUPPORTS_RESUME` or a native session). A `memory` need asks the runtime for +`SUPPORTS_MEMORY_SNAPSHOTS` and the harness for `Harness.SUPPORTS_MEMORY_RESTORE`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -68,7 +67,6 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", - "snapshots": "snapshots", "memory": "memory snapshots", } @@ -153,7 +151,6 @@ def supports(runtime: str, feature: str) -> bool: "user": "SUPPORTS_USER_SWITCHING", "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", - "snapshots": "SUPPORTS_FILESYSTEM_SNAPSHOTS", "memory": "SUPPORTS_MEMORY_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -184,8 +181,6 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" - if "snapshots" in needs and not cls.can_restore(): - return f"{name} cannot restore a snapshot" if "memory" in needs and not cls.SUPPORTS_MEMORY_RESTORE: return f"{name} cannot continue from a memory snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index e8e872159c..c1727e974b 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -1,167 +1,231 @@ -"""A retry restores the box's newest snapshot (checkpoint-restore-v1: the first attempt -dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns -before it, and finishes in a new harness process). +"""A retry restores the box's newest memory snapshot (memory-restore-v1: the reply of one +model turn fails once; the retry boots a new box from the memory snapshot of that turn, +and the harness program the box still runs gets the reply and finishes). -Every harness that can continue in a new process (`Harness.can_restore`) runs it on -prime, the one runtime with snapshots; the others are collected as skips. The retention policies run -on one harness: what they keep or delete does not depend on it. Every case checks that -the run leaks no snapshot: each one its boxes took is deleted on the platform once the -run is final.""" +Every harness that continues from a memory snapshot (`Harness.SUPPORTS_MEMORY_RESTORE`) +runs it on prime, the one runtime with memory snapshots. A process the task's `setup` +started holds a nonce only in its memory: the reward reads it from the restored box, so +a box that booted from files alone would fail. Every case checks that the run leaks no +snapshot.""" + +import asyncio +import pickle import pytest -from checkpoint_restore_v1 import FAIL_AFTER_TURNS +from conftest import _eval_config from grid import grid from prime_sandboxes import AsyncSandboxClient -from skill_codeword import SKILL -from verifiers.v1.harness import RESTORE_PROMPT -from verifiers.v1.runtimes import PrimeBackend -from verifiers.v1.types import content_text +from verifiers.v1.clients import ModelContext +from verifiers.v1.env import RunSlot +from verifiers.v1.restore import save_all +from verifiers.v1.runtimes import PrimeBackend, Snapshot +from verifiers.v1.utils.loaders import harness_class, load_environment RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} -NEEDS = ("snapshots", "mcp") - - -def checkpoint_env(**checkpoint) -> dict: - return {"agent": {"retries": RETRIES, "checkpoint": checkpoint}} +NEEDS = ("memory",) +SHELLS = ["bash", "rlm"] @pytest.fixture def taken(monkeypatch) -> list[str]: - """The ids of every snapshot the run's boxes take, in every attempt: also the - ones a restore rewinds past and the ones that never become restorable.""" - ids: list[str] = [] + """The id of every snapshot the run's boxes take, in every attempt.""" + snapshots: list[str] = [] snapshot = PrimeBackend.snapshot async def recording(self): result = await snapshot(self) - ids.append(result.id) + snapshots.append(result.id) return result monkeypatch.setattr(PrimeBackend, "snapshot", recording) - return ids + return snapshots -async def platform_state(snapshot_id: str) -> str | None: +async def platform_state(client: AsyncSandboxClient, snapshot_id: str) -> str | None: """The snapshot's state on prime, or None once it is gone.""" - async with AsyncSandboxClient() as client: - try: - return (await client.get_checkpoint(snapshot_id)).state - except Exception: # noqa: BLE001 - a deleted snapshot is not found - return None + try: + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None async def assert_no_leaks(taken: list[str]) -> None: - """Every snapshot the run took is deleted.""" - states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} - leaked = { - snapshot_id: state - for snapshot_id, state in states.items() - if state not in (None, "DELETING") - } + """Every snapshot the run took is deleted on the platform.""" + leaked = {} + async with AsyncSandboxClient() as client: + for snapshot_id in taken: + state = await platform_state(client, snapshot_id) + if state not in (None, "DELETING"): + leaked[snapshot_id] = state assert leaked == {} -@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) -async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): - """The box restores to the state before the turn that failed, the turns up to - there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not - rerun, and the harness is told about the restore — the restored attempt builds on - the earlier tool call's file. A snapshot every turn, so the failure at turn 3 - restores a snapshot with turns behind it.""" - trace = await restored_trace(run, harness, harness_runtime, interval=1) - assert trace.num_turns >= FAIL_AFTER_TURNS - await assert_no_leaks(taken) - - -@pytest.mark.parametrize( - "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True -) -async def test_a_short_rollout_takes_no_snapshot(run, taken, harness, harness_runtime): - """With the default interval (every 10th turn), a rollout that fails at turn 3 - has taken no snapshot: its retry starts over and still succeeds.""" +async def restored_trace( + run, harness, harness_runtime, *, fail_turn: int, **checkpoint +): (episode,) = await run( - "checkpoint-restore-v1", + "memory-restore-v1", harness=harness, runtime=harness_runtime, - env=checkpoint_env(), + taskset_overrides={"task": {"fail_turn": fail_turn}}, + env={"agent": {"retries": RETRIES, "checkpoint": checkpoint}}, max_turns=12, max_tokens=4096, ) (trace,) = episode.traces assert trace.ok, [error.message for error in trace.errors] + if not trace.errors: + # The harness handled the failed reply itself and finished its run: nothing + # failed, so nothing was restored. + pytest.skip(f"{harness.id} recovers from a failed model call by itself") + # The process `setup` started kept its memory across the restore. assert trace.reward == 1.0 assert [error.type for error in trace.errors] == ["ProviderError"] + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] assert trace.attempt == 2 - assert not trace.has_restored - assert trace.checkpoints == [] - assert taken == [] + assert trace.has_restored + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + # The harness program never restarted: one root. + assert sum(node.parent is None for node in trace.nodes) == 1 + # Every turn kept has its call; the failed turn's call stays, unlinked. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace -@pytest.mark.parametrize( - "harness,harness_runtime", - grid(["rlm"], needs=(*NEEDS, "skills")), - indirect=True, -) -async def test_rlm_with_skills_restores(run, taken, harness, harness_runtime): - """rlm keeps its session in a per-run skill home when it has skills: the retry - must load the session from the same home the restored box holds.""" - config = harness.model_copy(update={"skills": [SKILL]}) - trace = await restored_trace(run, config, harness_runtime, interval=1) - assert trace.num_turns >= FAIL_AFTER_TURNS +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): + """A snapshot at every turn. A harness with a shell fails at its second turn, so the + restored box holds the first turn's command and its harness goes on to the second; + a harness without one has a single turn, which fails.""" + shell = harness_class(harness.id).EXECUTES_CODE + trace = await restored_trace( + run, harness, harness_runtime, fail_turn=2 if shell else 1, interval=1 + ) + if shell: + assert trace.num_turns >= 2 + assert trace.metrics["copied"] == 1.0 + assert taken await assert_no_leaks(taken) -@pytest.mark.parametrize("keep", ["last", 2], ids=["keep-last", "keep-every-2nd"]) @pytest.mark.parametrize( - "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True ) -async def test_checkpoint_retention(run, taken, harness, harness_runtime, keep): - """Snapshots the policy drops are deleted while the rollout runs, without breaking - the newest one: the retry still restores it. Every snapshot is deleted once the - run is final.""" - trace = await restored_trace(run, harness, harness_runtime, interval=1, keep=keep) - assert all(checkpoint.deleted for checkpoint in trace.checkpoints) +async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_runtime): + """`save_all` checkpoints a running rollout for its owner. The rollout is then + cancelled with everything it holds, and its restore point goes through a pickle. A + second env, with a server, a tunnel and secrets of its own, continues the rollout + from the point: in a new box, with the harness program and the task's process + still running.""" + config = _eval_config( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": 99}}, + max_turns=12, + max_tokens=4096, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + first = load_environment(config.env) + (task,) = list(first.taskset) + async with first.serving(): + (slot,) = first.slots(task) + running = asyncio.ensure_future(first.run_slot(slot, ctx)) + try: + # Past its first turn: the box holds work the trace recorded. + async with asyncio.timeout(600): + while not (slot.traces and slot.traces[0].num_turns >= 1): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + turns_saved = len([node for node in point.trace["nodes"] if node["sampled"]]) + point = pickle.loads(pickle.dumps(point)) + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point.id + assert trace.reward == 1.0 + assert trace.metrics["copied"] == 1.0, [ + (n.message.role, n.sampled, str(n.message.model_dump(exclude_none=True))[:600]) + for n in trace.nodes + ] + assert trace.has_restored and trace.attempt == 2 + assert trace.num_turns > turns_saved >= 1 + assert sum(node.parent is None for node in trace.nodes) == 1 + # The owner holds the snapshot: no rollout deleted it. + (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] + assert snapshot_id == point.checkpoint + async with AsyncSandboxClient() as client: + assert (await client.get_checkpoint(snapshot_id)).state == "DURABLE" + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] await assert_no_leaks(taken) -async def restored_trace(run, harness, harness_runtime, **checkpoint): - (episode,) = await run( - "checkpoint-restore-v1", +@pytest.mark.parametrize( + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True +) +async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( + taken, harness, harness_runtime +): + """A restore point whose snapshot the platform no longer restores costs the rollout + no retry: it starts over under the same trace id and finishes.""" + config = _eval_config( + "memory-restore-v1", harness=harness, runtime=harness_runtime, - env=checkpoint_env(**checkpoint), + taskset_overrides={"task": {"fail_turn": 99}}, max_turns=12, max_tokens=4096, ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + first = load_environment(config.env) + (task,) = list(first.taskset) + async with first.serving(): + (slot,) = first.slots(task) + running = asyncio.ensure_future(first.run_slot(slot, ctx)) + try: + async with asyncio.timeout(600): + while not (slot.traces and slot.traces[0].num_turns >= 1): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + snapshot_id = point.snapshot["id"] + point.snapshot["id"] = "no-such-checkpoint" + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) (trace,) = episode.traces assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point.id assert trace.reward == 1.0 - assert [error.type for error in trace.errors] == ["ProviderError"] - # The newest snapshot before the failed turn boots the retry's box. - assert trace.agent.runtime is not None - restored_from = trace.agent.runtime.checkpoint - assert restored_from is not None - assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] - assert trace.has_restored - assert trace.attempt == 2 - # Each checkpoint records what its snapshot captured. - assert {c.snapshot.type for c in trace.checkpoints} == {"filesystem"} - # The error belongs to the first attempt, in the agent stage. - assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] - # The restored process made no side request of its own (it would start a second - # root), e.g. a model probe on session load. - assert sum(node.parent is None for node in trace.nodes) == 1 - # The restored process got the restore notice as a user turn. - assert any( - node.message.role == "user" - and RESTORE_PROMPT in content_text(node.message.content) - for node in trace.nodes - ) - # Every turn kept has its call; the dropped turns' calls stay, unlinked, so usage - # counts them. - assert [call.node for call in trace.calls if call.node is not None] == [ - i for i, node in enumerate(trace.nodes) if node.sampled - ] - assert any(call.node is None for call in trace.calls) - return trace + assert trace.attempt == 3 + + async with AsyncSandboxClient() as client, asyncio.timeout(300): + while (await client.get_checkpoint(snapshot_id)).state != "DURABLE": + await asyncio.sleep(2) + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] + await assert_no_leaks(taken) diff --git a/tests/integration/test_memory_checkpoint.py b/tests/integration/test_memory_checkpoint.py deleted file mode 100644 index d705965ccc..0000000000 --- a/tests/integration/test_memory_checkpoint.py +++ /dev/null @@ -1,271 +0,0 @@ -"""A retry restores the box's newest memory snapshot (memory-restore-v1: the reply of one -model turn fails once; the retry boots a new box from the memory snapshot of that turn, -and the harness program the box still runs gets the reply and finishes). - -Every harness that continues from a memory snapshot (`Harness.SUPPORTS_MEMORY_RESTORE`) -runs it on prime, the one runtime with memory snapshots. A process the task's `setup` -started holds a nonce only in its memory: the reward reads it from the restored box, so -a box that booted from the snapshot's files alone would fail. Every case checks that the -run leaks no snapshot.""" - -import asyncio -import pickle - -import pytest -from conftest import _eval_config -from grid import grid -from prime_sandboxes import AsyncSandboxClient - -from verifiers.v1.clients import ModelContext -from verifiers.v1.env import RunSlot -from verifiers.v1.harness import RESTORE_PROMPT -from verifiers.v1.restore import save_all -from verifiers.v1.runtimes import PrimeBackend, Snapshot -from verifiers.v1.types import content_text -from verifiers.v1.utils.loaders import harness_class, load_environment - -RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} -NEEDS = ("memory",) -SHELLS = ["bash", "rlm"] - - -@pytest.fixture -def taken(monkeypatch) -> list[tuple[str, str]]: - """The id and type of every snapshot the run's boxes take, in every attempt.""" - snapshots: list[tuple[str, str]] = [] - snapshot = PrimeBackend.snapshot - - async def recording(self, **kwargs): - result = await snapshot(self, **kwargs) - snapshots.append((result.id, result.type)) - return result - - monkeypatch.setattr(PrimeBackend, "snapshot", recording) - return snapshots - - -async def platform_state(client: AsyncSandboxClient, snapshot_id: str) -> str | None: - """The snapshot's state on prime, or None once it is gone.""" - try: - return (await client.get_checkpoint(snapshot_id)).state - except Exception: # noqa: BLE001 - a deleted snapshot is not found - return None - - -async def assert_no_leaks(taken: list[tuple[str, str]]) -> None: - """Every snapshot the run took is deleted on the platform.""" - leaked = {} - async with AsyncSandboxClient() as client: - for snapshot_id, _ in taken: - state = await platform_state(client, snapshot_id) - if state not in (None, "DELETING"): - leaked[snapshot_id] = state - assert leaked == {} - - -async def restored_trace( - run, harness, harness_runtime, *, fail_turn: int, **checkpoint -): - (episode,) = await run( - "memory-restore-v1", - harness=harness, - runtime=harness_runtime, - taskset_overrides={"task": {"fail_turn": fail_turn}}, - env={"agent": {"retries": RETRIES, "checkpoint": checkpoint}}, - max_turns=12, - max_tokens=4096, - ) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - if not trace.errors: - # The harness handled the failed reply itself and finished its run: nothing - # failed, so nothing was restored. - pytest.skip(f"{harness.id} recovers from a failed model call by itself") - # The process `setup` started kept its memory across the restore. - assert trace.reward == 1.0 - assert [error.type for error in trace.errors] == ["ProviderError"] - assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] - assert trace.attempt == 2 - assert trace.has_restored - assert trace.agent.runtime is not None - restored_from = trace.agent.runtime.checkpoint - assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] - assert {c.snapshot.type for c in trace.checkpoints} == {"memory"} - # The harness program never restarted: one root, and no restore notice. - assert sum(node.parent is None for node in trace.nodes) == 1 - assert not any( - node.message.role == "user" - and RESTORE_PROMPT in content_text(node.message.content) - for node in trace.nodes - ) - # Every turn kept has its call; the failed turn's call stays, unlinked. - assert [call.node for call in trace.calls if call.node is not None] == [ - i for i, node in enumerate(trace.nodes) if node.sampled - ] - assert any(call.node is None for call in trace.calls) - return trace - - -@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) -async def test_retry_restores_memory_checkpoint(run, taken, harness, harness_runtime): - """A snapshot at every turn. A harness with a shell fails at its second turn, so the - restored box holds the first turn's command and its harness goes on to the second; - a harness without one has a single turn, which fails.""" - shell = harness_class(harness.id).EXECUTES_CODE - trace = await restored_trace( - run, harness, harness_runtime, fail_turn=2 if shell else 1, interval=1 - ) - if shell: - assert trace.num_turns >= 2 - assert trace.metrics["copied"] == 1.0 - assert {kind for _, kind in taken} == {"memory"} - await assert_no_leaks(taken) - - -@pytest.mark.parametrize( - "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True -) -async def test_filesystem_snapshots_stay_selectable( - run, taken, harness, harness_runtime -): - """`snapshot = "filesystem"` takes no memory snapshot: the retry boots from the - snapshot's files, where the process that held the nonce is gone.""" - (episode,) = await run( - "memory-restore-v1", - harness=harness, - runtime=harness_runtime, - env={ - "agent": { - "retries": RETRIES, - "checkpoint": {"interval": 1, "snapshot": "filesystem"}, - } - }, - max_turns=12, - max_tokens=4096, - ) - (trace,) = episode.traces - assert trace.has_restored - assert {kind for _, kind in taken} == {"filesystem"} - assert trace.reward == 0.0 - await assert_no_leaks(taken) - - -@pytest.mark.parametrize( - "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True -) -async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_runtime): - """`save_all` checkpoints a running rollout for its owner. The rollout is then - cancelled with everything it holds, and its restore point goes through a pickle. A - second env, with a server, a tunnel and secrets of its own, continues the rollout - from the point: in a new box, with the harness program and the task's process - still running.""" - config = _eval_config( - "memory-restore-v1", - harness=harness, - runtime=harness_runtime, - taskset_overrides={"task": {"fail_turn": 99}}, - max_turns=12, - max_tokens=4096, - ) - ctx = ModelContext( - client=config.client, model=config.model, sampling=config.sampling - ) - - first = load_environment(config.env) - (task,) = list(first.taskset) - async with first.serving(): - (slot,) = first.slots(task) - running = asyncio.ensure_future(first.run_slot(slot, ctx)) - try: - # Past its first turn: the box holds work the trace recorded. - async with asyncio.timeout(600): - while not (slot.traces and slot.traces[0].num_turns >= 1): - assert not running.done(), "the rollout ended before the save" - await asyncio.sleep(0.2) - (point,) = await save_all() - finally: - running.cancel() - await asyncio.gather(running, return_exceptions=True) - turns_saved = len([node for node in point.trace["nodes"] if node["sampled"]]) - point = pickle.loads(pickle.dumps(point)) - - second = load_environment(config.env) - (task,) = list(second.taskset) - async with second.serving(): - episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - assert trace.id == point.id - assert trace.reward == 1.0 - assert trace.metrics["copied"] == 1.0, [ - (n.message.role, n.sampled, str(n.message.model_dump(exclude_none=True))[:600]) - for n in trace.nodes - ] - assert trace.has_restored and trace.attempt == 2 - assert trace.num_turns > turns_saved >= 1 - assert sum(node.parent is None for node in trace.nodes) == 1 - # The owner holds the snapshot: no rollout deleted it. - (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] - assert snapshot_id == point.checkpoint - async with AsyncSandboxClient() as client: - assert (await client.get_checkpoint(snapshot_id)).state == "DURABLE" - assert await PrimeBackend.discard([Snapshot(id=snapshot_id, type="memory")]) == [ - snapshot_id - ] - await assert_no_leaks(taken) - - -@pytest.mark.parametrize( - "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True -) -async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( - taken, harness, harness_runtime -): - """A restore point whose snapshot the platform no longer restores costs the rollout - no retry: it starts over under the same trace id and finishes.""" - config = _eval_config( - "memory-restore-v1", - harness=harness, - runtime=harness_runtime, - taskset_overrides={"task": {"fail_turn": 99}}, - max_turns=12, - max_tokens=4096, - ) - ctx = ModelContext( - client=config.client, model=config.model, sampling=config.sampling - ) - - first = load_environment(config.env) - (task,) = list(first.taskset) - async with first.serving(): - (slot,) = first.slots(task) - running = asyncio.ensure_future(first.run_slot(slot, ctx)) - try: - async with asyncio.timeout(600): - while not (slot.traces and slot.traces[0].num_turns >= 1): - assert not running.done(), "the rollout ended before the save" - await asyncio.sleep(0.2) - (point,) = await save_all() - finally: - running.cancel() - await asyncio.gather(running, return_exceptions=True) - snapshot_id = point.snapshot["id"] - point.snapshot["id"] = "no-such-checkpoint" - - second = load_environment(config.env) - (task,) = list(second.taskset) - async with second.serving(): - episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - assert trace.id == point.id - assert trace.reward == 1.0 - assert trace.attempt == 3 - - async with AsyncSandboxClient() as client, asyncio.timeout(300): - while (await client.get_checkpoint(snapshot_id)).state != "DURABLE": - await asyncio.sleep(2) - assert await PrimeBackend.discard([Snapshot(id=snapshot_id, type="memory")]) == [ - snapshot_id - ] - await assert_no_leaks(taken) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 3004316098..5f3f45326b 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -24,7 +24,6 @@ ) from verifiers.v1.types import ( AssistantMessage, - Request, ToolCall, ToolMessage, UserMessage, @@ -104,57 +103,25 @@ def test_agent_time_splits_off_the_checkpoint_hold(): assert trace.timing.agent.harness.duration == 15.0 -async def test_restored_tail_is_not_intercepted_twice(): - trace = make_trace() - seen: list[str] = [] - - def tag(request: Request) -> Request: - for message in request.messages: - if isinstance(message, ToolMessage | UserMessage): - seen.append(str(message.content)) - if isinstance(message, ToolMessage): - message.content = f"{message.content} [tagged]" - return request - - session = RolloutSession( - ctx=ModelContext(model="m", client=EvalClientConfig()), - trace=trace, - request_interceptors=[tag], - ) - tail = [ - ToolMessage(tool_call_id="c1", content="out [tagged]"), - UserMessage(content="go on"), - ] - session.mark_intercepted(trace.messages, tail) - request = Request(messages=[*trace.messages, *tail]) - rewritten, records, _ = await session.rewrite_request(request) - assert rewritten.messages == request.messages - assert records == [] - assert seen == [] - - class SlowBackend: - """Captures at once; becomes durable when the test says so.""" + """Stops the box for a snapshot until the test lets it run again; the snapshot + becomes durable when the test says so.""" def __init__(self) -> None: - self.SUPPORTS_FILESYSTEM_SNAPSHOTS = True self.durable = asyncio.Event() self.stopped = asyncio.Event() - """Set once a memory snapshot has stopped the box.""" + """Set once a snapshot has stopped the box.""" self.running = asyncio.Event() - """A memory snapshot returns when the test lets the box run again.""" + """A snapshot returns when this is set: the box runs again.""" self.fail = False self.taken = 0 self.deleted: list[str] = [] - async def snapshot(self, *, memory: bool = False) -> vf.Snapshot: + async def snapshot(self) -> vf.Snapshot: self.taken += 1 - if memory: - self.stopped.set() - await self.running.wait() - return vf.Snapshot( - id=f"ck{self.taken}", type="memory" if memory else "filesystem" - ) + self.stopped.set() + await self.running.wait() + return vf.Snapshot(id=f"ck{self.taken}") async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: await self.durable.wait() @@ -168,27 +135,50 @@ async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: return ids +class FakeRelay: + upstream = "https://tunnel.example" + + def __init__(self) -> None: + self.state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} + self.rerouted: list[str] = [] + + async def reroute(self, upstream: str) -> None: + self.rerouted.append(upstream) + + def make_rollout( - interval: int = 1, keep: Literal["last"] | int = "last" -) -> tuple[Rollout, SlowBackend]: + interval: int = 1, + keep: Literal["last"] | int = "last", + *, + running: bool = True, + stops: bool = False, +) -> tuple[Rollout, SlowBackend, RolloutSession]: + """A rollout that takes snapshots, with or without a harness segment that runs. + With `stops`, a snapshot keeps the box stopped until the test sets + `backend.running`.""" backend = SlowBackend() + if not stops: + backend.running.set() + job = {"job_id": "j1"} if running else None run = Rollout.__new__(Rollout) run.trace = make_trace() # one turn so far run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] - run.harness = SimpleNamespace(can_restore=lambda: True) # type: ignore[assignment] - - async def checkpoint_state() -> dict: - return {} - - run._harness_session = SimpleNamespace(checkpoint_state=checkpoint_state) # type: ignore[assignment] - run._session = SimpleNamespace(pinned_tool_results={}) # type: ignore[assignment] + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), trace=run.trace + ) + run._session = session + run._harness_session = SimpleNamespace( # type: ignore[assignment] + memory_state=lambda: None if job is None else {"job": job} + ) + run._relay = FakeRelay() # type: ignore[assignment] + run._secret = "box-secret" run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) run._durable = {} run._captures = set() run._closed = False run._opened = True - run._memory = False - run.unsaveable = "the 'fake' runtime takes no memory snapshots" + run._memory = True + run.unsaveable = None run.save_skipped = None run._held = False run._release_timer = None @@ -197,33 +187,30 @@ async def checkpoint_state() -> dict: run.deleted_snapshots = set() run.rewound_snapshots = [] run.restored_turns = None - return run, backend - - -TURN = SimpleNamespace(prefix_node_ids=[0, 1], tail=[]) + return run, backend, session async def test_checkpoint_interval_skips_turns(): - run, backend = make_rollout(interval=2) - assert await run._checkpoint(TURN) is not None # turn 2: the 2nd + run, backend, _ = make_rollout(interval=2) + assert await run._checkpoint() is not None # turn 2: the 2nd run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) run.trace.nodes.append( MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) ) - assert await run._checkpoint(TURN) is None # turn 3 + assert await run._checkpoint() is None # turn 3 assert backend.taken == 1 async def test_the_restored_turn_is_not_snapshotted_again(): - run, backend = make_rollout(interval=2) + run, backend, _ = make_rollout(interval=2) run.restored_turns = run.trace.num_turns - assert await run._checkpoint(TURN) is None + assert await run._checkpoint() is None assert backend.taken == 0 async def test_checkpoint_becomes_durable_in_the_background(): - run, backend = make_rollout(interval=1) - checkpoint = await run._checkpoint(TURN) + run, backend, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint() assert checkpoint is not None and checkpoint.captured assert checkpoint.time.end == 0 and checkpoint.bytes is None backend.durable.set() @@ -233,8 +220,8 @@ async def test_checkpoint_becomes_durable_in_the_background(): async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): - run, backend = make_rollout(interval=1) - await run._checkpoint(TURN) + run, backend, _ = make_rollout(interval=1) + await run._checkpoint() backend.fail = True backend.durable.set() await run._settle_checkpoints(wait=True) @@ -242,16 +229,16 @@ async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): async def test_settling_without_a_retry_stops_waiting(): - run, _ = make_rollout(interval=1) - checkpoint = await run._checkpoint(TURN) + run, _, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint() await asyncio.wait_for(run._settle_checkpoints(wait=False), timeout=1) assert run._durable == {} assert run.trace.checkpoints == [checkpoint] and checkpoint.time.end == 0 async def test_a_closing_run_takes_no_checkpoint(): - run, backend = make_rollout(interval=1) - capture = asyncio.create_task(run._checkpoint(TURN)) + run, backend, _ = make_rollout(interval=1) + capture = asyncio.create_task(run._checkpoint()) run._closed = True await run._settle_checkpoints(wait=True) assert await capture is None @@ -285,11 +272,11 @@ def test_keep_policies(): async def test_a_durable_snapshot_prunes_the_older_ones(): - run, backend = make_rollout(keep="last") + run, backend, _ = make_rollout(keep="last") run.trace.checkpoints = durable("old1", "old2") run.rewound_snapshots = [vf.Snapshot(id="rewound")] backend.durable.set() - newest = await run._checkpoint(TURN) + newest = await run._checkpoint() await run._settle_checkpoints(wait=True) assert sorted(backend.deleted) == ["old1", "old2", "rewound"] assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] @@ -320,50 +307,18 @@ def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] assert warnings == [ ( - "rollout checkpointing is off: the 'subprocess' runtime takes no " + "rollout checkpointing is off: the 'subprocess' runtime takes no memory " "snapshots; a failed rollout retries from the start" ), ( - "rollout checkpointing is off: harness 'terminus-2' does not support it; " - "a failed rollout retries from the start" + "rollout checkpointing is off: harness 'terminus-2' cannot continue from " + "a memory snapshot; a failed rollout retries from the start" ), ] -class FakeRelay: - upstream = "https://tunnel.example" - - def __init__(self) -> None: - self.state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} - self.rerouted: list[str] = [] - - async def reroute(self, upstream: str) -> None: - self.rerouted.append(upstream) - - -def make_memory_rollout( - running: bool = True, -) -> tuple[Rollout, SlowBackend, RolloutSession]: - """A rollout that takes memory snapshots, with or without a harness segment that - runs.""" - job = {"job_id": "j1"} if running else None - run, backend = make_rollout() - session = RolloutSession( - ctx=ModelContext(model="m", client=EvalClientConfig()), trace=run.trace - ) - run._session = session - run._memory = True - run.unsaveable = None - run._relay = FakeRelay() # type: ignore[assignment] - run._secret = "box-secret" - run._harness_session = SimpleNamespace( # type: ignore[assignment] - memory_state=lambda: None if job is None else {"job": job} - ) - return run, backend, session - - -async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): - run, backend, session = make_memory_rollout() +async def test_a_snapshot_holds_every_reply_until_the_box_runs_again(): + run, backend, session = make_rollout(stops=True) session.idempotent_requests = { "retry:/v1/chat/completions:aa": IdempotentRequest( binding=("/v1/chat/completions", b"aa"), @@ -377,7 +332,7 @@ async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): ), "open": IdempotentRequest(binding=("/v1/chat/completions", b"cc")), } - capture = asyncio.create_task(run._checkpoint(TURN)) + capture = asyncio.create_task(run._checkpoint()) await backend.stopped.wait() # The box is stopped: nothing it asked for may be answered. assert not session.flowing.is_set() @@ -387,7 +342,7 @@ async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): assert session.flowing.is_set() # The stop killed the box's connections: its relay opens new ones. assert run._relay.rerouted == ["https://tunnel.example"] - assert checkpoint.snapshot == vf.Snapshot(id="ck1", type="memory") + assert checkpoint.snapshot == vf.Snapshot(id="ck1") assert checkpoint.node == len(run.trace.nodes) - 1 assert checkpoint.live == { "program": {"job": {"job_id": "j1"}}, @@ -400,24 +355,24 @@ async def test_a_memory_snapshot_holds_every_reply_until_the_box_runs_again(): async def test_no_memory_snapshot_without_a_running_segment(): - run, backend, session = make_memory_rollout(running=False) - assert await run._checkpoint(TURN) is None + run, backend, session = make_rollout(running=False) + assert await run._checkpoint() is None assert backend.taken == 0 and session.flowing.is_set() async def test_a_failed_memory_snapshot_lets_replies_flow_again(): - run, backend, session = make_memory_rollout() + run, backend, session = make_rollout() - async def failing(*, memory: bool = False): + async def failing(): raise vf.SandboxError("pause refused") backend.snapshot = failing # type: ignore[method-assign] - assert await run._checkpoint(TURN) is None + assert await run._checkpoint() is None assert session.flowing.is_set() and run.trace.checkpoints == [] async def test_save_returns_a_restore_point_another_process_can_open(): - run, backend, session = make_memory_rollout() + run, backend, session = make_rollout() session.pinned_tool_results = { ("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned") } @@ -431,7 +386,7 @@ async def test_save_returns_a_restore_point_another_process_can_open(): point = pickle.loads(pickle.dumps(point)) assert point.id == run.trace.id - assert point.snapshot == {"id": "ck1", "type": "memory"} + assert point.snapshot == {"id": "ck1"} task = vf.Task(vf.TaskData(idx=0, prompt="q")) trace, restored = point.open(task) assert trace.id == run.trace.id and trace.num_turns == run.trace.num_turns @@ -446,7 +401,7 @@ async def test_save_returns_a_restore_point_another_process_can_open(): async def test_save_returns_before_the_snapshot_is_durable(): """The owner gets the restore point once the box runs again; the snapshot becomes durable in the background, and the rollout's teardown waits for it.""" - run, backend, _ = make_memory_rollout() + run, backend, _ = make_rollout() backend.running.set() point = await run.save() (checkpoint,) = run.trace.checkpoints @@ -457,21 +412,21 @@ async def test_save_returns_before_the_snapshot_is_durable(): async def test_a_held_save_answers_the_box_only_after_release(): - run, backend, session = make_memory_rollout() + run, backend, session = make_rollout() backend.running.set() assert await run.save(hold=True) is not None assert not session.flowing.is_set() # A checkpoint of the run's own under the hold does not end it. - await run._checkpoint(TURN) + await run._checkpoint() assert not session.flowing.is_set() run.release() assert session.flowing.is_set() and run._release_timer is None async def test_a_failed_held_save_does_not_hold_the_box(): - run, backend, session = make_memory_rollout() + run, backend, session = make_rollout() - async def failing(*, memory: bool = False): + async def failing(): raise vf.SandboxError("pause refused") backend.snapshot = failing # type: ignore[method-assign] @@ -481,14 +436,15 @@ async def failing(*, memory: bool = False): async def test_a_rollout_without_memory_snapshots_saves_nothing(): - run, backend = make_rollout() + run, backend, _ = make_rollout() + run._memory, run.unsaveable = False, "the 'fake' runtime takes no memory snapshots" assert await run.save() is None assert backend.taken == 0 assert run.save_skipped == "the 'fake' runtime takes no memory snapshots" async def test_a_save_between_segments_names_its_reason(): - run, _, _ = make_memory_rollout(running=False) + run, _, _ = make_rollout(running=False) assert await run.save() is None assert run.save_skipped == "its harness program is not running (setup or scoring)" @@ -515,7 +471,7 @@ def reason(harness: str, runtime: vf.RuntimeConfig, **config) -> str | None: async def test_an_owners_snapshot_outlives_the_rollout(): """A pinned snapshot is not under the keep policy, is not deleted with the run's own, and becomes durable even when the run settles without a retry to wait for.""" - run, backend, _ = make_memory_rollout() + run, backend, _ = make_rollout() backend.running.set() saving = asyncio.create_task(run.save()) await asyncio.sleep(0) @@ -529,26 +485,8 @@ async def test_an_owners_snapshot_outlives_the_rollout(): assert await saving is not None (pinned,) = run.trace.checkpoints # A newer snapshot of the run's own does not prune the owner's. - newest = await run._checkpoint(TURN) + newest = await run._checkpoint() await run._settle_checkpoints(wait=True) assert backend.deleted == [] assert kept(run.trace.checkpoints, "last") == {newest.id} assert not pinned.deleted - - -async def test_filesystem_snapshots_at_turns_leave_saves_to_the_owner(): - """`snapshot = "filesystem"` picks the snapshot of the checkpoints at model turns; - an owner's save still takes a memory snapshot.""" - run, backend, _ = make_memory_rollout() - run._checkpointing = RolloutCheckpointConfig(interval=1, snapshot="filesystem") - - async def checkpoint_state() -> dict: - return {} - - run._harness_session.checkpoint_state = checkpoint_state - backend.running.set() - backend.durable.set() - at_turn = await run._checkpoint(TURN) - assert at_turn.snapshot.type == "filesystem" - point = await run.save() - assert point.snapshot["type"] == "memory" diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index bd1b66a992..af5a67fe4d 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -17,7 +17,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.errors import HarnessError, InterceptionError -from verifiers.v1.harness import RESTORE_PROMPT, Harness, HarnessSession +from verifiers.v1.harness import Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess from verifiers.v1.runtimes.base import current_program from verifiers.v1.semantic import ( @@ -26,7 +26,7 @@ ) from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace -from verifiers.v1.types import Messages, UserMessage +from verifiers.v1.types import Messages from verifiers.v1.utils.aio import run_shielded ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() @@ -77,17 +77,8 @@ class ACPConfig: class ACPHarness(Harness[ConfigT]): """Harness backed by one live ACP process and native session per rollout.""" - LOADS_SESSIONS: ClassVar[bool] = False - """Whether the pinned agent implements ACP `session/load`, so a new process can - continue a session from its files in the box. An agent that does not advertise - `loadSession` at runtime takes no snapshots either way.""" - SUPPORTS_MEMORY_RESTORE: ClassVar[bool] = True - @classmethod - def can_restore(cls) -> bool: - return cls.LOADS_SESSIONS - async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script( ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"} @@ -255,10 +246,6 @@ def __init__( self._responses: dict[int, asyncio.Future] = {} self._request_id = 0 self._response_task: asyncio.Task | None = None - self._session_id: str | None = None - """The agent's ACP session id, once a checkpoint asked for it.""" - self._restore_id: str | None = None - """The session the next process loads from the restored box (`restore`).""" self._detached: list[str] | None = None """The runner's argv once it runs as a job of the box (`memory`): each request is then a job too, since no connection from the host survives a stopped box.""" @@ -400,28 +387,6 @@ async def _request(self, operation: str, **payload: Any) -> JsonObject: if not future.done(): future.cancel() - async def checkpoint_state(self) -> dict[str, Any] | None: - # The agent keeps its session in files of the box, so a restored box can load - # it in a new process — if the agent advertises `loadSession`. - if self._session_id is None: - if not self._running: - return None - response = await self._request("session") - result = response.get("result") - if not response.get("ok") or not isinstance(result, dict): - raise RuntimeError(response.get("error") or "ACP session query failed") - if not result.get("load_session") or result.get("session_id") is None: - return None - self._session_id = result["session_id"] - return {"acp_session_id": self._session_id} - - def restore(self, state: dict[str, Any], tail: Messages) -> Messages: - # The agent's own session files hold the conversation, the tail included as - # far as it flushed them; the restored process takes only the notice. - self._restore_id = state["acp_session_id"] - self._session_id = self._restore_id - return [UserMessage(content=RESTORE_PROMPT)] - async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed: raise RuntimeError("ACP session is closed") @@ -464,7 +429,6 @@ async def _run(self, messages: Messages | None) -> ProgramResult: ) config = { "command": self.config.command, - "load_session_id": self._restore_id, "user_contents": user_contents, "mcp_urls": self.mcp_urls, "mcp_headers": self.harness.config.resolve_mcp_headers(self.mcp_urls), @@ -491,10 +455,6 @@ async def _run(self, messages: Messages | None) -> ProgramResult: except BaseException: await run_shielded(self._stop(graceful=False)) raise - finally: - # Only the restored box's first process loads the session; later - # turns continue the live one. - self._restore_id = None return self._finish(response, calls_before) def _finish(self, response: JsonObject, calls_before: int) -> ProgramResult: diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index 0fc1830b40..2cf91e3357 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -87,8 +87,6 @@ def __init__(self) -> None: self.stop_reason: str | None = None self.response_metadata: dict[str, Any] = {} self.update_metadata: list[dict[str, Any]] = [] - self.loading = False - """Set while `session/load` replays the history, which no turn reports.""" def reset(self) -> None: self.visible_reply = "" @@ -106,8 +104,6 @@ def turn_result(self) -> ACPTurn: ) async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: - if self.loading: - return metadata = dict(kwargs) if isinstance(field_meta := getattr(update, "field_meta", None), dict): metadata.update(field_meta) @@ -271,37 +267,18 @@ async def start(self, config: dict) -> None: self.supports_steering = (initialized.field_meta or {}).get( "steering", {} ).get("supported") is True - load_id = config.get("load_session_id") - if load_id is None: - session = await self.connection.new_session( - cwd=os.getcwd(), - mcp_servers=mcp_servers(config), - **config["session_meta"], - ) - session_id = session.session_id - else: - # A box restored from a checkpoint: the agent's session files are - # here, its process is not. Load the session in this new process. - if not (self.capabilities and self.capabilities.load_session): - raise RuntimeError("ACP agent does not support session/load") - self.client.loading = True - try: - await self.connection.load_session( - cwd=os.getcwd(), - session_id=load_id, - mcp_servers=mcp_servers(config), - **config["session_meta"], - ) - finally: - self.client.loading = False - session_id = load_id + session = await self.connection.new_session( + cwd=os.getcwd(), + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) except BaseException: with suppress(BaseException): await self.stack.aclose() self._reset() raise - self.session_id = session_id - self.is_new = load_id is None + self.session_id = session.session_id + self.is_new = True async def run(self, config: dict) -> ACPTurn: gate = config.get("tool_interception") @@ -403,13 +380,6 @@ async def handle(self, request: dict) -> dict: result = await session.steer( request["message"], request.get("message_id") ) - elif operation == "session": - result = { - "session_id": session.session_id, - "load_session": bool( - session.capabilities and session.capabilities.load_session - ), - } elif operation == "shutdown": if self.tasks: await asyncio.gather(*self.tasks, return_exceptions=True) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d8129efd62..fcb7790cac 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -35,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.restore import RestorePoint, restoring +from verifiers.v1.restore import RestorePoint, restoring, unsaveable from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, @@ -662,20 +662,12 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: config = self.config.checkpoint if config is None: return None - harness = self.harness - backend = backend_cls(self.config.runtime) - if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: - return config - if harness.SUPPORTS_MEMORY_RESTORE and backend.SUPPORTS_MEMORY_SNAPSHOTS: + reason = unsaveable(self.config, self.harness) + if reason is None: return config - key = (harness.config.id, self.config.runtime.type) + key = (self.harness.config.id, self.config.runtime.type) if key not in _checkpointing_warned: _checkpointing_warned.add(key) - reason = ( - f"harness {harness.config.id!r} does not support it" - if not harness.can_restore() - else f"the {self.config.runtime.type!r} runtime takes no snapshots" - ) logger.warning( "rollout checkpointing is off: %s; a failed rollout retries from the " "start", @@ -810,9 +802,9 @@ async def run( and trace.num_turns <= run.restored_turns ) ): - # The restore led nowhere: the box never booted from it, or the - # attempt failed before its first new turn (a harness that - # cannot load its session fails the same way every time). The + # The restore led nowhere: the box never ran from it, or the + # attempt failed before its first new turn (what the box was + # doing at the snapshot fails the same way every time). The # next retry steps back to an older snapshot, or starts over. failed_restores.add(target.id) if trace.ok: diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index e7815c020e..8f3fead15f 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -40,22 +40,14 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing: snapshot the box every n-th model turn, so a retry - continues from the newest snapshot instead of starting over. Each checkpoint - records the type of its snapshot. A memory snapshot holds the box's processes, so - the restored box continues the harness program itself; it needs a runtime that - takes them (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host + """Rollout checkpointing: snapshot the box's files and memory every n-th model + turn, so a retry continues from the newest snapshot instead of starting over. The + restored box continues the harness program itself. A snapshot stops the box for a + few seconds. It needs a runtime that takes memory snapshots + (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services - of its own. A filesystem snapshot holds the box's files; it needs a runtime that - takes them (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new process can - continue the rollout (`Harness.can_restore`). Elsewhere a retry starts over. Every - snapshot is deleted once the run is final.""" - - snapshot: Literal["memory", "filesystem"] = "memory" - """The snapshot to take at model turns: `memory` where the rollout supports it and - a filesystem snapshot elsewhere, or always `filesystem`. A memory snapshot stops - the box for a few seconds; a filesystem snapshot does not. An owner's save - (`save_all`) takes a memory snapshot either way.""" + of its own. Elsewhere a retry starts over. Every snapshot is deleted once the run + is final.""" interval: PositiveInt = 10 """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index ba2eb54bd6..dd4a3d56b8 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -18,7 +18,7 @@ from verifiers.v1.runtimes import Program, ProgramResult, Runtime from verifiers.v1.runtimes.base import current_program from verifiers.v1.task import TaskData -from verifiers.v1.types import Message, Messages, UserMessage +from verifiers.v1.types import Message, Messages from verifiers.v1.utils.decorators import ( discover_decorated, invoke_all, @@ -41,16 +41,6 @@ ConfigT = TypeVar("ConfigT", bound=HarnessConfig) -RESTORE_PROMPT = ( - "Your workspace was restored from a filesystem checkpoint taken before your last " - "model turn, and you are continuing in a new process. Files are as they were at " - "the checkpoint. Background processes, running shells, sub-agents, and in-memory " - "state such as REPL variables from before the restore are gone; start again " - "whatever you still need. Your last action may not have taken effect, so check " - "before you rely on it. Continue the task." -) -"""The user turn a harness gets after a restore (`HarnessSession.restore`).""" - class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False @@ -84,14 +74,6 @@ class Harness(ABC, Generic[ConfigT]): program runs as a job of the box that the host only waits for (`Runtime.run(..., program=)`), and it reaches the host only through the endpoint it was given.""" - @classmethod - def can_restore(cls) -> bool: - """Whether a new process of this harness can continue a rollout in a box - restored from a snapshot (`AgentConfig.checkpoint`): the default relaunches on - the trace's transcript, so a harness that resumes that way (`SUPPORTS_RESUME`) - can.""" - return cls.SUPPORTS_RESUME - def __init__(self, config: ConfigT) -> None: self.config = config @@ -465,21 +447,6 @@ async def _run(self, messages: Messages | None) -> ProgramResult: self.tool_interception_url, ) - async def checkpoint_state(self) -> dict[str, Any] | None: - """What `restore` needs to continue in a box restored from a checkpoint taken - now, or None when this session cannot be restored (no checkpoint is taken). - The default relaunches on the trace's transcript and needs nothing.""" - return {} - - def restore(self, state: dict[str, Any], tail: Messages) -> Messages: - """Prepare this new session to continue a rollout in a box restored from a - checkpoint, and return the messages of its first segment. `state` is what - `checkpoint_state` returned when the checkpoint was taken; `tail` is the - checkpoint's uncommitted prompt after the trace's branch. The default - relaunches on the branch plus `tail`, then tells the program about the - restore (`RESTORE_PROMPT`).""" - return [*tail, UserMessage(content=RESTORE_PROMPT)] - async def close(self) -> None: """Close session-owned resources. Idempotent.""" self._closed = True diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index f4b7462916..3287be9994 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -19,7 +19,7 @@ ACP_VERSION = "0.99.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" -STATE_ROOT = "/tmp/vf-claude-code-runs" +CLAUDE_CONFIG_ROOT = ".vf-claude" ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -42,7 +42,6 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed @@ -105,10 +104,6 @@ async def prepare_acp( "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"), "ANTHROPIC_API_KEY": secret, "ANTHROPIC_MODEL": ctx.model, - # Claude Code validates a model it has not seen in this process with a - # one-token "Hi" request (e.g. when a restored session re-selects its - # model); a custom model option is trusted without that request. - "ANTHROPIC_CUSTOM_MODEL_OPTION": ctx.model, "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions), "CLAUDE_CONFIG_DIR": config_dir, "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", @@ -140,4 +135,4 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def config_dir(trace: Trace) -> str: - return f"{STATE_ROOT}/{trace.id}" + return f"{CLAUDE_CONFIG_ROOT}/{trace.id}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 3b28c64fcd..8fc16e543b 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,7 +22,6 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" -STATE_ROOT = "/tmp/vf-codex-runs" ACP_VERSION = "2.5.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" @@ -63,7 +62,6 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True - LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: @@ -176,7 +174,7 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def trace_home(trace: Trace) -> str: - return f"{STATE_ROOT}/{trace.id}" + return f"/tmp/vf-codex-home-{trace.id}" async def build_env( self, diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 4051875607..81c259f227 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -31,8 +31,8 @@ BuiltinTool = Literal["bash", "edit", "fetch", "ipython"] RLM_REPO = "github.com/PrimeIntellect-ai/nano-rlm.git" -RLM_CACHE_DIR = "/var/tmp/vf-rlm" -STATE_ROOT = "/tmp/vf-rlm-runs" +RLM_CACHE_DIR = "/tmp/vf-rlm" +RLM_STATE_DIR = "/tmp/vf-rlm-state" RLM_RUNTIME_METADATA_KEY = "ai.prime.rlm/runtime-v1" RLM_SESSION_METADATA_KEY = "ai.prime.rlm/session-v1" @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="a057f58d144384fed6ecfed1dabcf350fe540910", min_length=1 + default="ae1cc01f8293987d18b09286b34d93c0dfbf8dac", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" @@ -142,18 +142,12 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - LOADS_SESSIONS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - # A retry runs the same harness and keeps the directory: it holds - # `RLM_HOME`, whose session a box restored from a checkpoint loads. - if self._skills_install_dir is None: - self._skills_install_dir = ( - f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" - ) + self._skills_install_dir = f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" await self.install_skills(runtime, f"{self._skills_home()}/skills") directory = self._install_dir() binary = f"{directory}/bin/rlm" @@ -312,14 +306,14 @@ def acp_close_result(self, trace: Trace, response_metadata: dict[str, Any]) -> N self._consume_snapshot(trace, response_metadata) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - await runtime.run(["rm", "-rf", f"{STATE_ROOT}/{trace.id}"], {}) + await runtime.run(["rm", "-rf", f"{RLM_STATE_DIR}/{trace.id}"], {}) if self._skills_install_dir is not None: await remove_dir( runtime, self._skills_install_dir, "RLM skill installation" ) def _home(self, trace: Trace) -> str: - return self._skills_home() or f"{STATE_ROOT}/{trace.id}/home" + return self._skills_home() or f"{RLM_STATE_DIR}/{trace.id}/home" def _skills_home(self) -> str | None: """rlm's home for a run that carries skills: they sit in its `skills/`, where diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 328fc0ad6b..6d22092658 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1040,7 +1040,7 @@ async def sample() -> web.Response: # before the reply returns matches the prompt exactly (tool results in, this # reply not yet). It runs alongside the model call; the reply waits for it. capture = ( - asyncio.create_task(session.checkpointer(turn)) + asyncio.create_task(session.checkpointer()) if session.checkpointer is not None else None ) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 38f5b0c9c1..480cdefdd7 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -12,7 +12,6 @@ from typing import Literal from urllib.parse import urlsplit -from verifiers.v1 import graph from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import AgentConfig, RolloutCheckpointConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig @@ -53,7 +52,7 @@ hook_boundary, ) from verifiers.v1.state import state_cls -from verifiers.v1.task import Task, TaskData +from verifiers.v1.task import Task from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage from verifiers.v1.utils.aio import run_shielded @@ -207,7 +206,6 @@ def __init__( """The model turns the restored trace held when this run began (None for a run that did not restore). The next turn is the one the snapshot was taken at, which this run does not snapshot again.""" - self._harness_data: TaskData | None = None self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" self._pruning = asyncio.Lock() @@ -280,7 +278,7 @@ def __init__( """Whether an owner holds the box's replies past its save (`save(hold=True)`).""" self._release_timer: asyncio.TimerHandle | None = None self._capturing = asyncio.Lock() - if self._restore is not None and self._restore.snapshot.type == "memory": + if self._restore is not None: # Set before the box boots: its relay repeats the requests that waited as # soon as it reaches this session. self._session.pinned_tool_results.update(self._restore.pins) @@ -400,9 +398,7 @@ async def open(self) -> bool: self.harness.config.name, self.runtime_config.type, ) - memory_restore = ( - self._restore is not None and self._restore.snapshot.type == "memory" - ) + restored = self._restore is not None loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -470,8 +466,7 @@ async def open(self) -> bool: self.trace.timing.boot.end = now self.trace.timing.setup.start = now self.trace.notify() - # A restored box already holds the task's setup; only the harness's - # (idempotent) install runs again. + # A restored box already holds the task's setup. if self._restore is None: started = time.perf_counter() async with boundary(TaskError, "task setup"): @@ -499,13 +494,11 @@ async def open(self) -> bool: self._relay = Relay( runtime, self.trace.id, - running=self._restore.live["relay"] - if memory_restore - else None, + running=self._restore.live["relay"] if restored else None, ) self._stack.push_async_callback(self._relay.stop) - # A box restored from memory still runs the harness. - if not memory_restore: + # A restored box still runs the harness. + if not restored: await self.harness.setup(self._agent_runtime) self.unsaveable = unsaveable(self.trace.agent.config, self.harness) or ( "it runs in a borrowed box" @@ -519,20 +512,14 @@ async def open(self) -> bool: else None ) self._memory = self.unsaveable is None and self._relay is not None - if memory_restore and not self._memory: + if restored and not self._memory: raise SandboxError( "the rollout cannot continue from its memory snapshot" ) if ( self._checkpointing is not None and self._borrowed_runtime is None - and ( - self._memory - or ( - runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - and self.harness.can_restore() - ) - ) + and self._memory ): self._session.checkpointer = self._checkpoint logger.debug( @@ -572,7 +559,7 @@ async def open(self) -> bool: # A harness restored from memory still sends the secret it got at its # launch; the relay swaps it for this session's. self._secret = ( - self._restore.live["secret"] if memory_restore else model_secret + self._restore.live["secret"] if restored else model_secret ) if self._relay is not None: # The harness holds the model secret anyway; the relay gets @@ -728,7 +715,6 @@ async def open(self) -> bool: ) else {} ) - self._harness_data = harness_data self._harness_session = await self.harness.session( self.ctx, self.trace, @@ -796,15 +782,12 @@ async def step( self, messages: Messages | None = None, *, - intercepted: int = 0, on_input: Callable[[], None] | None = None, ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - The first `intercepted` messages pass to the harness as they are — they were - intercepted when the trace first saw them (a restored checkpoint's tail). Returns whether the exchange can continue — a refused turn (limit, @stop), a failure (an expired agent timeout included), or a segment that made no progress all end it.""" @@ -826,15 +809,11 @@ async def step( try: async with deadline: assert self._harness_session is not None - if ( - messages is not None - and len(messages) > intercepted - and self._session.request_interceptors - ): + if messages is not None and self._session.request_interceptors: prepared, rewrites = await self._session.prepare_users( - Request(messages=messages[intercepted:]) + Request(messages=messages) ) - messages = [*messages[:intercepted], *prepared.messages] + messages = prepared.messages self.trace.request_rewrites.extend(rewrites) if self._session.stopped: return False @@ -914,28 +893,15 @@ async def _turn(self, messages: Messages | None) -> bool: return False async def restore(self) -> bool: - """Run the restored attempt's first segment: a new harness process in the box - restored from the checkpoint continues the conversation the checkpoint left - off at (`HarnessSession.restore`), told that the box was restored.""" + """Run the restored attempt's first segment: the box restored from the + checkpoint still runs the harness program, so this waits for it + (`HarnessSession.reattach`) instead of starting one.""" checkpoint = self._restore - assert checkpoint is not None and self._harness_data is not None - assert self._harness_session is not None - if checkpoint.snapshot.type == "memory": - # The box still runs the segment: wait for it instead of starting one. - self._harness_session.reattach(checkpoint.live["program"]) - return await self.step() - # The relaunch re-emits the session's system prompt (the task's, plus a - # network notice) through the program's own system message; the tail's copy - # would double it. - system = self._harness_data.system_prompt - tail = [m for m in checkpoint.tail if m.role != "system" or system is None] - self._session.pinned_tool_results.update(checkpoint.pins) - self._session.mark_intercepted(self.trace.messages, tail) - messages = self._harness_session.restore(checkpoint.harness, tail) - # Only the restore notice is new; user hooks prepare it like any user turn. - return await self.step(messages, intercepted=len(messages) - 1) + assert checkpoint is not None and self._harness_session is not None + self._harness_session.reattach(checkpoint.live["program"]) + return await self.step() - async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: + async def _checkpoint(self) -> Checkpoint | None: if self._closed: # Settling may have drained `_captures` already: a snapshot taken now # would start its upload after the box is gone. @@ -944,74 +910,18 @@ async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: if task is not None: self._captures.add(task) try: - return await self._capture(turn) + return await self._capture() finally: self._captures.discard(task) - async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: - assert self.runtime is not None + async def _capture(self) -> Checkpoint | None: assert self._checkpointing is not None turn_number = self.trace.num_turns + 1 if turn_number % self._checkpointing.interval or ( self.restored_turns is not None and turn_number == self.restored_turns + 1 ): return None - if self._memory and self._checkpointing.snapshot == "memory": - return await self._capture_memory() - if not ( - self.runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - and self.harness.can_restore() - ): - return None - if len(self.trace._pending) > 1: - # Another request is in flight (parallel sub-agents share the trace): the - # box is not idle, so no snapshot would match this turn's prefix. - return - assert self._harness_session is not None - # Read before the first await: the reply may commit and run its hooks while - # the harness answers, and the checkpoint must describe the idle box. - node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None - tail = list(turn.tail) - state = copy.deepcopy(self.trace.state) - info = copy.deepcopy(self.trace.info) - pins = dict(self._session.pinned_tool_results) - try: - harness = await self._harness_session.checkpoint_state() - except Exception: - logger.warning( - "harness checkpoint state failed (rollout %s)", - self.trace.id, - exc_info=True, - ) - return - if harness is None: - return - started = time.time() - try: - snapshot = await self.runtime.backend.snapshot() - except Exception: - logger.warning("snapshot failed (rollout %s)", self.trace.id, exc_info=True) - return None - checkpoint = Checkpoint( - id=snapshot.id, - node=node, - tail=tail, - state=state, - info=info, - pins=pins, - harness=harness, - snapshot=snapshot, - time=TimeSpan(start=started), - captured=time.time(), - ) - self.trace.checkpoints.append(checkpoint) - self._durable[checkpoint.id] = asyncio.create_task( - self._make_durable(checkpoint) - ) - logger.debug( - "checkpoint %s taken: id=%s node=%s", checkpoint.id, self.trace.id, node - ) - return checkpoint + return await self._take_snapshot() async def save(self, *, hold: bool = False) -> RestorePoint | None: """Take a memory checkpoint of the running rollout now and return its restore @@ -1038,7 +948,7 @@ async def save(self, *, hold: bool = False) -> RestorePoint | None: self._captures.add(task) self._held = hold try: - checkpoint = await self._capture_memory(pinned=True) + checkpoint = await self._take_snapshot(pinned=True) finally: self._captures.discard(task) if checkpoint is None: @@ -1060,7 +970,7 @@ def release(self) -> None: self._release_timer = None self._session.resume() - async def _capture_memory(self, *, pinned: bool = False) -> Checkpoint | None: + async def _take_snapshot(self, *, pinned: bool = False) -> Checkpoint | None: """Snapshot the box with its memory. Nothing the box asked for is answered while the snapshot is taken (`RolloutSession.hold`), so the box it stops has seen exactly the trace as recorded here; a box restored from it asks again @@ -1091,7 +1001,7 @@ async def _capture_memory(self, *, pinned: bool = False) -> Checkpoint | None: } started = time.time() try: - snapshot = await self.runtime.backend.snapshot(memory=True) + snapshot = await self.runtime.backend.snapshot() except Exception: logger.warning( "memory snapshot failed (rollout %s)", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 606214333e..239d39b6b0 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -216,7 +216,6 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True - SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = True SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: @@ -395,32 +394,26 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e - async def snapshot(self, *, memory: bool = False) -> Snapshot: - # The platform fixes the snapshot's content before the request returns, then - # uploads it. Until it is durable the snapshot is stray: the platform refuses - # to delete it while this box lives, so teardown frees it after. + async def snapshot(self) -> Snapshot: + # A pause is the platform's memory snapshot: it stops the box, and the resume + # continues it. The platform uploads the snapshot after the pause. Until it is + # durable the snapshot is stray: the platform refuses to delete it while this + # box lives, so teardown frees it after. async def create() -> Snapshot: - if memory: - # A pause is the platform's memory snapshot: it stops the box, and the - # resume continues it. Shielded as one: a box left paused never runs - # its rollout again. - self._awake.clear() + # Shielded as one: a box left paused never runs its rollout again. + self._awake.clear() + try: + paused = await self._client.pause(self.info.id) + snapshot = Snapshot(id=paused.checkpoint_id) + self.stray_snapshots.append(snapshot) try: - paused = await self._client.pause(self.info.id) - snapshot = Snapshot(id=paused.checkpoint_id, type="memory") - self.stray_snapshots.append(snapshot) - try: - await self._settle("PAUSED", "PAUSE_FAILED") - except BaseException: - self.stray_snapshots.remove(snapshot) - raise - await self._resume() - finally: - self._awake.set() - return snapshot - checkpoint = await self._client.checkpoint(self.info.id) - snapshot = Snapshot(id=checkpoint.id) - self.stray_snapshots.append(snapshot) + await self._settle("PAUSED", "PAUSE_FAILED") + except BaseException: + self.stray_snapshots.remove(snapshot) + raise + await self._resume() + finally: + self._awake.set() return snapshot try: @@ -526,10 +519,7 @@ async def discard(cls, snapshots: list[Snapshot]) -> list[str]: @classmethod def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: return config.model_copy( - update={ - "checkpoint": snapshot.id, - "restore_memory": snapshot.type == "memory", - } + update={"checkpoint": snapshot.id, "restore_memory": True} ) async def prepare_execution( diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index e0125d7cb0..a695a0054b 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -14,7 +14,7 @@ from collections.abc import AsyncIterator, Mapping from contextvars import ContextVar from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar from pydantic import BaseModel from pydantic_config import BaseConfig @@ -145,15 +145,12 @@ class BaseRuntimeInfo(BaseConfig): class Snapshot(BaseModel): - """A snapshot of a box that new boxes can boot from (`RuntimeBackend.snapshot`, - `restore_config`), as many times as needed.""" + """A snapshot of a box's files and memory (`RuntimeBackend.snapshot`). A new box + made from it (`restore_config`) continues every process where the snapshot stopped + it; any number of boxes can.""" id: str """The provider's id for the snapshot.""" - type: Literal["filesystem", "memory"] = "filesystem" - """What the snapshot captures, as the runtime took it: `filesystem` holds the box's - files, not its processes; `memory` holds its files and its memory, so a box that - boots from it continues every process where the snapshot stopped it.""" @dataclass @@ -222,14 +219,11 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" - SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False - """Whether `snapshot` captures the box's filesystem for new boxes to boot from - (`restore_config`). Rollout checkpointing (`AgentConfig.checkpoint`) takes one at - model turns, so a retry continues from there instead of starting over.""" - SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = False - """Whether `snapshot(memory=True)` captures the box's memory with its filesystem, so - a new box continues the processes of the old one (`attach`).""" + """Whether `snapshot` captures the box's memory with its files, so a new box + continues the processes of the old one (`restore_config`, `attach`). Rollout + checkpointing (`AgentConfig.checkpoint`) takes one at model turns, so a retry + continues from there instead of starting over.""" info: BaseRuntimeInfo @@ -268,12 +262,10 @@ def type(self) -> str: async def start(self) -> None: pass - async def snapshot(self, *, memory: bool = False) -> Snapshot: - """Snapshot the box and return it once its content is fixed: writes after this - returns are not in it. It becomes restorable only later (`wait_snapshot`); the - box keeps running. With `memory` the snapshot holds the box's memory too: the - box stops while it is taken and runs again when this returns, and its - connections to the outside may be dead.""" + async def snapshot(self) -> Snapshot: + """Snapshot the box's files and memory. The box stops while the snapshot is + taken and runs again when this returns; its connections to the outside may be + dead by then. The snapshot becomes restorable only later (`wait_snapshot`).""" raise NotImplementedError(f"{type(self).__name__} does not support snapshots") async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: @@ -303,8 +295,8 @@ async def discard(cls, snapshots: list[Snapshot]) -> list[str]: @classmethod def restore_config(cls, config: Any, snapshot: Snapshot) -> Any: - """The runtime config for a new box that boots from `snapshot` instead of the - task's image.""" + """The runtime config for a new box that continues `snapshot` instead of + booting the task's image.""" raise NotImplementedError(f"{cls.__name__} does not support snapshots") async def stop(self) -> None: diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index aa98addd49..493464bac0 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -227,12 +227,10 @@ class RolloutSession: last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and their parse, which the next request's equal prefix reuses instead of parsing again.""" - checkpointer: Callable[[graph.PendingTurn], Awaitable[Checkpoint | None]] | None = ( - None - ) + checkpointer: Callable[[], Awaitable[Checkpoint | None]] | None = None """Snapshots the rollout's box as a turn goes upstream, alongside the model call; - the reply waits for the capture (`Rollout` sets it when its runtime supports - checkpoints and a retry may restore one). Returns the checkpoint it took, if any.""" + the reply waits for the capture (`Rollout` sets it when the rollout takes + snapshots). Returns the checkpoint it took, if any.""" flowing: asyncio.Event = field(default_factory=asyncio.Event, repr=False) """Cleared while a memory snapshot of the box is taken (`hold`): nothing the box @@ -424,21 +422,6 @@ async def rewrite_request( self.pinned_tool_results[tool_keys[position]] = after return current, records, None - def mark_intercepted(self, branch: Messages, tail: Messages) -> None: - """Treat `tail`, which follows `branch` and was intercepted in an earlier - attempt (a restored checkpoint's), as prepared: the harness re-sends it, and - a non-idempotent hook must not rewrite it twice.""" - request = Request(messages=[*branch, *tail]) - for position, key in self.tool_result_keys(request).items(): - message = request.messages[position] - if position >= len(branch) and isinstance(message, ToolMessage): - self.pinned_tool_results.setdefault(key, message) - self.prepared_users.update( - graph.message_hash(message) - for message in tail - if isinstance(message, UserMessage) - ) - def consume_prepared(self, messages: Messages) -> None: """Forget pre-harness user rewrites only after their model request commits.""" for message in messages: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 6429e83841..75aff17e79 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -376,21 +376,17 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall class Checkpoint(BaseModel): - """A snapshot of the rollout's box taken as a model turn went upstream: the box holds - the effects of every message in `node`'s conversation plus `tail`, and nothing of the - turn that followed. A retry restores it (`Trace.rewind`).""" + """A snapshot of the rollout's running box, memory included: the box holds the + effects of `node`'s conversation and still runs the harness program. A retry + restores it (`Trace.rewind`) into a new box that continues every process.""" id: str """The snapshot's id.""" snapshot: Snapshot - """The box's snapshot (`RuntimeBackend.snapshot`) a retry boots a new box from.""" + """The box's snapshot (`RuntimeBackend.snapshot`) a retry continues a new box from.""" node: int | None = None - """The last committed node whose conversation the box's state follows; None when - the snapshot preceded the first turn.""" - tail: Messages = Field(default_factory=list, exclude=True) - """The turn's prompt messages after `node` that had no node yet (tool results, user - turns): with `node`'s conversation, the prompt a restored harness continues from. - Live-only — a retry restores it in the process that took the snapshot.""" + """The last committed node when the snapshot was taken; None before the first + turn. The restored box asks again for whatever was open after it.""" state: Any = Field(None, exclude=True) """The trace's typed state as the snapshot was taken (tool servers write it over `/state`), restored with the box so a retry sees one consistent world. Live-only.""" @@ -398,14 +394,14 @@ class Checkpoint(BaseModel): """`Trace.info` as the snapshot was taken, restored with `state`. Live-only.""" pins: dict[tuple[str, str], ToolMessage] = Field(default_factory=dict, exclude=True) """The rollout's pinned tool results as the snapshot was taken - (`RolloutSession.pinned_tool_results`): a restored harness re-sends its own - copies, which must be pinned again to match the trace. Live-only.""" + (`RolloutSession.pinned_tool_results`): the restored box re-sends its own copies, + which must be pinned again to match the trace. Live-only.""" live: dict[str, Any] = Field(default_factory=dict, exclude=True) - """What continues the box of a memory snapshot: the harness program's job + """What continues the restored box: the harness program's job (`HarnessSession.memory_state`), the box's relay and the secret the harness holds. Live-only.""" replays: dict[str, Any] = Field(default_factory=dict, exclude=True) - """The replies the server had just sent when a memory snapshot stopped the box + """The replies the server had just sent when the snapshot stopped the box (`RolloutSession.idempotent_requests`): the restored box asks again for one that had not arrived, and gets the reply the trace already holds. Live-only.""" time: TimeSpan = Field(default_factory=TimeSpan) @@ -418,9 +414,6 @@ class Checkpoint(BaseModel): longer, since the snapshot runs alongside it.""" bytes: int | None = None """The snapshot's stored size, which the provider bills, once it is durable.""" - harness: dict[str, Any] = Field(default_factory=dict) - """What the harness session needs to continue in a box restored from the snapshot - (`HarnessSession.checkpoint_state`), e.g. an ACP agent's session id.""" restored: bool = False """Whether a retry restored the run from this snapshot.""" pinned: bool = False From 3b4db58019445f244ba6740456173cbcd944a65d Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 22:24:36 +0000 Subject: [PATCH 37/53] snapshot a rollout when its model call fails, not at turns A snapshot at every 10th turn stops every healthy box for a few seconds. Take the snapshot when it is needed instead: when a model call fails with an error the run retries, before the harness gets the error. The box that the retry restores still waits for the reply, and asks for the call again. `checkpoint.interval` is now off by default. It stays as an option for failures that leave no box to snapshot (the box died, the harness program crashed). A model call that fails again right after a restore no longer counts as a restore that led nowhere: the retry restores the same snapshot. Co-Authored-By: Claude --- docs/agent.md | 10 ++++- tests/integration/test_checkpoint.py | 29 ++++++++++---- tests/unit/test_checkpoint.py | 52 +++++++++++++++++++++++++ tests/unit/test_retry_ids.py | 6 ++- verifiers/v1/agent.py | 12 ++++-- verifiers/v1/configs/agent.py | 17 ++++---- verifiers/v1/interception/server.py | 17 ++++++++ verifiers/v1/rollout.py | 43 +++++++++++++++++++-- verifiers/v1/session.py | 4 ++ verifiers/v1/utils/retries.py | 58 ++++++++++++++++++---------- 10 files changed, 204 insertions(+), 44 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index dac83f6c22..344cb8ab10 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -79,6 +79,12 @@ skip_on_timeout = true Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. +A rollout takes a snapshot only when it needs one: + +- When a model call fails with an error that the run retries (`[env.agent.retries]`). The snapshot is taken before the harness gets the error, so the box that the retry restores still waits for the reply and asks for the call again. A rollout that never fails takes no snapshot and is never stopped. +- When its owner saves it (`save_all`, below). +- With `interval`, also at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. Without it, a retry after such a failure starts over. + A snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. A rollout takes snapshots only when all of these hold. Otherwise a retry starts over, with a warning once per harness and runtime: @@ -88,11 +94,11 @@ A rollout takes snapshots only when all of these hold. Otherwise a retry starts - The rollout runs in a box of its own, not in a borrowed one. - The rollout has no tool servers, no shared tools and no services of its own. They reach the host at an address that does not follow the box to its restore. -A retry also starts over when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +A retry also starts over when a restore fails before the restored attempt's first new turn, unless a model call failed again. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] -interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none +# interval = 10 # also snapshot at the 10th, 20th, ... model turn (off by default) keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index c1727e974b..899606fab0 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -1,6 +1,7 @@ """A retry restores the box's newest memory snapshot (memory-restore-v1: the reply of one -model turn fails once; the retry boots a new box from the memory snapshot of that turn, -and the harness program the box still runs gets the reply and finishes). +model turn fails once and the box is snapshotted before the harness gets the error; the +retry continues a new box from that snapshot, and the harness program the box still runs +gets the reply and finishes). Every harness that continues from a memory snapshot (`Harness.SUPPORTS_MEMORY_RESTORE`) runs it on prime, the one runtime with memory snapshots. A process the task's `setup` @@ -100,17 +101,31 @@ async def restored_trace( @pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): - """A snapshot at every turn. A harness with a shell fails at its second turn, so the - restored box holds the first turn's command and its harness goes on to the second; - a harness without one has a single turn, which fails.""" + """The one snapshot of the run is taken when the model call fails. A harness with + a shell fails at its second turn, so the restored box holds the first turn's + command and its harness goes on to the second; a harness without one has a single + turn, which fails.""" shell = harness_class(harness.id).EXECUTES_CODE trace = await restored_trace( - run, harness, harness_runtime, fail_turn=2 if shell else 1, interval=1 + run, harness, harness_runtime, fail_turn=2 if shell else 1 ) if shell: assert trace.num_turns >= 2 assert trace.metrics["copied"] == 1.0 - assert taken + assert len(taken) == 1 + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_snapshots_at_turns_are_restored_too( + run, taken, harness, harness_runtime +): + """With `interval`, the box is also snapshotted at model turns. The failed call + takes no second snapshot of the same turn: the retry restores the one of turn 2.""" + trace = await restored_trace(run, harness, harness_runtime, fail_turn=2, interval=1) + assert [c.id for c in trace.checkpoints if c.restored] == [taken[1]] await assert_no_leaks(taken) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 5f3f45326b..42355de4d5 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -12,6 +12,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.configs.retries import RetryConfig from verifiers.v1.graph import MessageNode from verifiers.v1.restore import RestorePoint, unsaveable from verifiers.v1.rollout import Rollout, kept @@ -22,6 +23,7 @@ TimeSpan, TraceTask, ) +from verifiers.v1.utils.retries import RetryState from verifiers.v1.types import ( AssistantMessage, ToolCall, @@ -173,6 +175,8 @@ def make_rollout( run._relay = FakeRelay() # type: ignore[assignment] run._secret = "box-secret" run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) + run._retryable = lambda error: isinstance(error, vf.ProviderError) + run.rescued = False run._durable = {} run._captures = set() run._closed = False @@ -294,8 +298,56 @@ def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: ) +async def test_no_snapshot_at_turns_by_default(): + run, backend, _ = make_rollout() + run._checkpointing = RolloutCheckpointConfig() + assert await run._checkpoint() is None + assert backend.taken == 0 + + +async def test_a_failed_model_call_that_the_run_retries_takes_a_snapshot(): + run, backend, session = make_rollout() + run._checkpointing = RolloutCheckpointConfig() + # The run would not retry this error: the box is not stopped for it. + await run._rescue(vf.HarnessError("exit 1")) + assert backend.taken == 0 + await run._rescue(vf.ProviderError("overloaded", status_code=529)) + (checkpoint,) = run.trace.checkpoints + assert backend.taken == 1 and checkpoint.node == len(run.trace.nodes) - 1 + assert run.rescued + assert session.flowing.is_set() + # The same call fails again: the snapshot already holds the box that waits for it. + await run._rescue(vf.ProviderError("overloaded", status_code=529)) + assert backend.taken == 1 + # A turn later, a new failure needs a new snapshot. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + await run._rescue(vf.ProviderError("overloaded", status_code=529)) + assert backend.taken == 2 + + +async def test_a_run_without_a_retry_left_takes_no_snapshot(): + run, backend, _ = make_rollout() + run._retryable = None + await run._rescue(vf.ProviderError("overloaded", status_code=529)) + assert backend.taken == 0 + + +def test_retry_state_tells_whether_an_error_gets_a_retry_without_spending_it(): + state = RetryState( + RetryConfig(max_retries=0, rules=[{"type": "ProviderError", "max_retries": 1}]) + ) + error = vf.ProviderError("overloaded", status_code=529) + assert state.allows(error) and state.allows(error) + assert not state.allows(vf.HarnessError("exit 1")) + trace = make_trace() + trace.record_error(error) + assert state.next_error(trace.errors) is not None + assert not state.allows(error) + + def test_checkpointing_is_on_by_default(): assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() + assert RolloutCheckpointConfig().interval is None assert gated("bash", vf.PrimeConfig())._checkpointing() is not None diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 3a046037bc..71e4343e8e 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -39,6 +39,7 @@ class FakeRollout: def __init__(self, *, on_trace, trace_id, attempt, **_): self.rewound_snapshots: list[str] = [] self.deleted_snapshots: set[str] = set() + self.rescued = False self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) @@ -69,8 +70,8 @@ async def close(self) -> vf.Trace: @pytest.mark.asyncio async def test_a_restore_that_leads_nowhere_is_not_picked_again(monkeypatch): """Attempt 1 takes a snapshot and fails. Attempt 2 restores it and fails before its - first new turn, as a harness that cannot load its session would every time. The - next retry starts over instead of restoring the same snapshot again.""" + first new turn, with an error that no failed model call explains. The next retry + starts over instead of restoring the same snapshot again.""" restores: list[str | None] = [] class FakeRollout: @@ -80,6 +81,7 @@ def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): restores.append(None if restore is None else restore[1].id) self.rewound_snapshots: list[vf.Snapshot] = [] self.deleted_snapshots: set[str] = set() + self.rescued = False self.restored_turns = None if restore is None: self.trace = make_trace(id=trace_id, attempt=attempt) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 30827b364f..8636efe78b 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -794,6 +794,8 @@ async def run( attempt=first_attempt + attempt + 1, restore=restoring_from, checkpoint=checkpointing, + # The retry after this attempt is still to be granted. + retryable=(retry_state.allows if attempt < budget else None), ) trace = run.trace snapshots.update((s.id, s) for s in run.rewound_snapshots) @@ -808,14 +810,16 @@ async def run( any(error.stage == "setup" for error in trace.errors[seen:]) or ( not trace.ok + and not run.rescued and run.restored_turns is not None and trace.num_turns <= run.restored_turns ) ): # The restore led nowhere: the box never ran from it, or the - # attempt failed before its first new turn (what the box was - # doing at the snapshot fails the same way every time). The - # next retry steps back to an older snapshot, or starts over. + # attempt failed before its first new turn. The next retry + # steps back to an older snapshot, or starts over. A model + # call that failed again is not the snapshot's fault: its + # retry restores the same box (`Rollout.rescued`). failed_restores.add(target.id) if trace.ok: break @@ -917,6 +921,7 @@ async def _run_once( attempt: int, restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, + retryable: Callable[[Exception], bool] | None = None, ) -> Rollout: if collect_artifacts: _check_artifact_runtime(params["runtime_config"]) @@ -928,6 +933,7 @@ async def _run_once( trace_id=trace_id, restore=restore, checkpoint=checkpoint, + retryable=retryable, attempt=attempt, **params, ) diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 8f3fead15f..9575922fc7 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -40,18 +40,21 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing: snapshot the box's files and memory every n-th model - turn, so a retry continues from the newest snapshot instead of starting over. The - restored box continues the harness program itself. A snapshot stops the box for a - few seconds. It needs a runtime that takes memory snapshots + """Rollout checkpointing: snapshot the box's files and memory when a model call + fails with an error the run retries, so the retry continues from there instead of + starting over. The snapshot is taken before the harness gets the error: the + restored box continues the harness program itself and asks for the call again. A + rollout that never fails takes no snapshot. A snapshot stops the box for a few + seconds. It needs a runtime that takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services of its own. Elsewhere a retry starts over. Every snapshot is deleted once the run is final.""" - interval: PositiveInt = 10 - """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter - than n turns takes none, and a restore loses fewer than n turns.""" + interval: PositiveInt | None = None + """Also snapshot at every n-th model turn (the n-th, the 2n-th, ...), for failures + that leave no box to snapshot (the box died, the harness program crashed): a retry + then loses fewer than n turns instead of starting over. None takes none.""" keep: Literal["last"] | PositiveInt = "last" """Snapshots kept while the rollout runs: the newest (`"last"`), or every k-th plus the newest (`k`; 1 keeps every one). A failed restore falls back to an older kept diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 6d22092658..3a4e07e170 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1044,6 +1044,21 @@ async def sample() -> web.Response: if session.checkpointer is not None else None ) + + async def rescue(failure: Exception) -> None: + """Snapshot the box before the harness learns of `failure`: a retry + then continues a box that still waits for this turn's reply.""" + if session.rescuer is None: + return + if capture is not None: + await asyncio.wait({capture}) + try: + await session.rescuer(failure) + except Exception: + logger.warning( + "checkpoint failed: id=%s", session.trace.id, exc_info=True + ) + try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1120,6 +1135,7 @@ async def sample() -> web.Response: logger.warning( "Model call failed - %s: %s", type(e).__name__, clip(str(e)) ) + await rescue(e) return _error_response( dialect, str(e), status=getattr(e, "status_code", 502) ) @@ -1128,6 +1144,7 @@ async def sample() -> web.Response: logger.warning( "Model call failed - %s: %s", type(e).__name__, clip(str(e)) ) + await rescue(e) return _error_response(dialect, str(e), status=502) except BaseException as e: # A cancelled exchange (harness disconnect, shutdown) is still diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 480cdefdd7..d09b3c5f4b 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -167,6 +167,7 @@ def __init__( attempt: int = 1, restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, + retryable: Callable[[Exception], bool] | None = None, ) -> None: """`attempt` numbers this run among the trace's retries, from 1 (stamped on the trace and the errors it captures). `restore` continues a failed attempt's trace from one of @@ -196,6 +197,11 @@ def __init__( self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts self._checkpointing = checkpoint + self._retryable = retryable + """Whether the run's owner would retry it after this error (`Agent.run`).""" + self.rescued = False + """Whether the newest snapshot holds the box as it waited for a model call that + then failed (`_rescue`): a retry restores it and asks for the call again.""" self._durable: dict[str, asyncio.Task[None]] = {} self._captures: set[asyncio.Task] = set() """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" @@ -522,6 +528,7 @@ async def open(self) -> bool: and self._memory ): self._session.checkpointer = self._checkpoint + self._session.rescuer = self._rescue logger.debug( "Set up %s harness in %s", self.harness.config.name, @@ -916,13 +923,43 @@ async def _checkpoint(self) -> Checkpoint | None: async def _capture(self) -> Checkpoint | None: assert self._checkpointing is not None + interval = self._checkpointing.interval turn_number = self.trace.num_turns + 1 - if turn_number % self._checkpointing.interval or ( - self.restored_turns is not None and turn_number == self.restored_turns + 1 + if ( + interval is None + or turn_number % interval + or ( + self.restored_turns is not None + and turn_number == self.restored_turns + 1 + ) ): return None return await self._take_snapshot() + async def _rescue(self, error: Exception) -> None: + """Snapshot the box whose model call just failed with `error`, if a retry of + the run would follow. The box still waits for the call's reply, so the box a + retry restores asks for it again.""" + if self._closed or self._retryable is None or not self._retryable(error): + return + newest = next( + (c for c in reversed(self.trace.checkpoints) if not c.deleted), None + ) + if newest is not None and newest.node == self._last_node(): + # No turn since the newest snapshot: it holds this same open call. + self.rescued = True + return + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + self.rescued = await self._take_snapshot() is not None + finally: + self._captures.discard(task) + + def _last_node(self) -> int | None: + return len(self.trace.nodes) - 1 if self.trace.nodes else None + async def save(self, *, hold: bool = False) -> RestorePoint | None: """Take a memory checkpoint of the running rollout now and return its restore point, for an owner that continues the rollout in another process @@ -987,7 +1024,7 @@ async def _take_snapshot(self, *, pinned: bool = False) -> Checkpoint | None: program = self._harness_session.memory_state() if program is None: return None - node = len(self.trace.nodes) - 1 if self.trace.nodes else None + node = self._last_node() state = copy.deepcopy(self.trace.state) info = copy.deepcopy(self.trace.info) pins = dict(session.pinned_tool_results) diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 493464bac0..a550b5e652 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -232,6 +232,10 @@ class RolloutSession: the reply waits for the capture (`Rollout` sets it when the rollout takes snapshots). Returns the checkpoint it took, if any.""" + rescuer: Callable[[Exception], Awaitable[None]] | None = None + """Snapshots the rollout's box when a model call failed, before the harness gets + the error (`Rollout` sets it when the rollout takes snapshots).""" + flowing: asyncio.Event = field(default_factory=asyncio.Event, repr=False) """Cleared while a memory snapshot of the box is taken (`hold`): nothing the box asked for is answered or written until it is set again, so the box the snapshot diff --git a/verifiers/v1/utils/retries.py b/verifiers/v1/utils/retries.py index 7870325d45..e93bd82ec8 100644 --- a/verifiers/v1/utils/retries.py +++ b/verifiers/v1/utils/retries.py @@ -95,26 +95,44 @@ def next_error(self, errors: Iterable[Error]) -> Error | None: Return the triggering error so logs identify the actual retry cause. """ for error in errors: - for index, rule in enumerate(self.rules): - if rule.type is not None and error.type != rule.type: - continue - if rule.status_code is not None and ( - error.status_code is None - or not any( - error.status_code == status - if isinstance(status, int) - else error.status_code // 100 == int(status[0]) - for status in rule.status_code - ) - ): - continue - pattern = self.patterns[index] - if pattern is not None and pattern.search(error.message) is None: - continue - if self.used[index] < rule.max_retries: - self.used[index] += 1 - return error - break + index = self._rule(error) + if index is not None and self.used[index] < self.rules[index].max_retries: + self.used[index] += 1 + return error + return None + + def allows(self, error: Exception) -> bool: + """Whether `next_error` would grant a retry for `error` now. Consumes nothing.""" + from verifiers.v1.trace import Error + + index = self._rule( + Error( + type=type(error).__name__, + message=str(error), + status_code=getattr(error, "status_code", None), + ) + ) + return index is not None and self.used[index] < self.rules[index].max_retries + + def _rule(self, error: Error) -> int | None: + """The first rule that matches `error`: it alone sets the error's budget.""" + for index, rule in enumerate(self.rules): + if rule.type is not None and error.type != rule.type: + continue + if rule.status_code is not None and ( + error.status_code is None + or not any( + error.status_code == status + if isinstance(status, int) + else error.status_code // 100 == int(status[0]) + for status in rule.status_code + ) + ): + continue + pattern = self.patterns[index] + if pattern is not None and pattern.search(error.message) is None: + continue + return index return None From c6c38541b81e5104431e4fb48d38fe70004af901 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 22:32:28 +0000 Subject: [PATCH 38/53] sort imports Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 42355de4d5..03ec373c72 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -23,13 +23,13 @@ TimeSpan, TraceTask, ) -from verifiers.v1.utils.retries import RetryState from verifiers.v1.types import ( AssistantMessage, ToolCall, ToolMessage, UserMessage, ) +from verifiers.v1.utils.retries import RetryState def make_trace() -> vf.Trace: From c85ae3e3de0a3e70366034ec939d1e88d5f6af48 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 11 Nov 2026 21:08:24 +0000 Subject: [PATCH 39/53] test the restore of a box with two running sub-agents A memory snapshot holds every process of the box, so the sub-agents of a harness continue with it. Nothing tested that. The new case saves an rlm rollout while two sub-agents are at work, and checks that a second owner's box continues both without starting either again. Co-Authored-By: Claude --- .../integration/fixtures/subagent_restore.py | 108 ++++++++++++++++++ tests/integration/test_checkpoint.py | 101 +++++++++++----- 2 files changed, 180 insertions(+), 29 deletions(-) create mode 100644 tests/integration/fixtures/subagent_restore.py diff --git a/tests/integration/fixtures/subagent_restore.py b/tests/integration/fixtures/subagent_restore.py new file mode 100644 index 0000000000..cf917f89e6 --- /dev/null +++ b/tests/integration/fixtures/subagent_restore.py @@ -0,0 +1,108 @@ +"""subagent-restore: a task whose agent hands two jobs to two sub-agents that run at the +same time, so a box snapshotted in the middle holds two running sub-agents. + +`setup` starts one process per sub-agent that holds a nonce only in its memory and +hands it to whoever reads its FIFO. Each sub-agent reads the nonce twice, one command +per turn. The second command first waits for the other sub-agent's first file and then +sleeps, so from the moment both sub-agents have a turn on the trace, neither finishes +for `HOLD_SECONDS`: a snapshot taken then catches both at work. The second read comes +from the process, so it works only in a box that kept its memory. +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +NAMES = ("alpha", "bravo") +ROOT_MARK = "COORDINATOR-TASK" +HOLD_SECONDS = 20 +SERVER = 'rm -f "$2"; mkfifo "$2"; while :; do printf %s "$1" > "$2"; sleep 0.5; done' + + +def mark(name: str) -> str: + """The text only `name`'s brief holds.""" + return f"SUBAGENT-{name.upper()}" + + +def brief(name: str) -> str: + other = NAMES[1 - NAMES.index(name)] + return ( + f"{mark(name)}. Run these shell commands in the working directory, one " + "command per turn, and wait for each result before the next.\n" + f"1. `timeout 10 cat {name}.fifo > {name}.1`\n" + f"2. `timeout 120 sh -c 'until [ -f {other}.1 ]; do sleep 1; done'; " + f"sleep {HOLD_SECONDS}; timeout 10 cat {name}.fifo > {name}.done`\n" + "3. Reply with the single word: done." + ) + + +def sub_agent_turns(trace: vf.Trace) -> dict[str, list[int]]: + """The sampled nodes of each sub-agent: those whose history holds the sub-agent's + brief and not the coordinator's task.""" + turns: dict[str, list[int]] = {name: [] for name in NAMES} + for index, node in enumerate(trace.nodes): + if not node.sampled: + continue + history, cursor = "", node.parent + while cursor is not None: + history += str(trace.nodes[cursor].message.model_dump()) + cursor = trace.nodes[cursor].parent + if ROOT_MARK in history: + continue + for name in NAMES: + if mark(name) in history: + turns[name].append(index) + return turns + + +class SubagentRestoreTask(vf.Task): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + for name in NAMES: + nonce = uuid.uuid4().hex + trace.info[name] = nonce + await runtime.run_background( + ["sh", "-c", SERVER, "sh", nonce, f"{name}.fifo"], {}, f"{name}.log" + ) + + async def _read(self, runtime: Runtime, command: str) -> str: + try: + result = await runtime.run(["sh", "-c", command], {}) + except (SandboxError, OSError): + return "" + return result.stdout.strip() + + @vf.reward(weight=1.0) + async def both_done(self, trace: vf.Trace, runtime: Runtime) -> float: + """Each sub-agent's second read left its nonce in its done file.""" + done = [ + await self._read(runtime, f"cat {name}.done") == trace.info[name] + for name in NAMES + ] + return float(all(done)) + + +class SubagentRestoreTaskset(vf.Taskset[SubagentRestoreTask, vf.TasksetConfig]): + def load(self) -> list[SubagentRestoreTask]: + briefs = "\n\n".join( + f"Brief for sub-agent {i + 1}:\n{brief(name)}" + for i, name in enumerate(NAMES) + ) + return [ + SubagentRestoreTask( + vf.TaskData( + idx=0, + prompt=( + f"{ROOT_MARK}. Do not run the commands below yourself. Start " + "exactly two sub-agents so that both run at the same time, " + "and give each one its brief word for word. Wait for both, " + "then reply with the single word: done.\n\n" + f"{briefs}" + ), + ) + ) + ] + + +__all__ = ["SubagentRestoreTaskset"] diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index 899606fab0..4835b5403e 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -16,6 +16,7 @@ from conftest import _eval_config from grid import grid from prime_sandboxes import AsyncSandboxClient +from subagent_restore import sub_agent_turns from verifiers.v1.clients import ModelContext from verifiers.v1.env import RunSlot @@ -26,6 +27,8 @@ RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} NEEDS = ("memory",) SHELLS = ["bash", "rlm"] +# rlm with its guidance on how to hand work to sub-agents. +SUB_AGENTS = {"id": "rlm", "delegation_prompt": True} @pytest.fixture @@ -99,6 +102,27 @@ async def restored_trace( return trace +async def saved_rollout(config, ctx, ready=lambda trace: trace.num_turns >= 1): + """Run the config's one rollout until `ready(trace)`, checkpoint it with `save_all`, + and cancel it with everything it holds. Returns its slot and its restore point. By + default the rollout is past its first turn: the box holds work the trace recorded.""" + env = load_environment(config.env) + (task,) = list(env.taskset) + async with env.serving(): + (slot,) = env.slots(task) + running = asyncio.ensure_future(env.run_slot(slot, ctx)) + try: + async with asyncio.timeout(600): + while not (slot.traces and ready(slot.traces[0])): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + return slot, point + + @pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): """The one snapshot of the run is taken when the model call fails. A harness with @@ -150,21 +174,7 @@ async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_r client=config.client, model=config.model, sampling=config.sampling ) - first = load_environment(config.env) - (task,) = list(first.taskset) - async with first.serving(): - (slot,) = first.slots(task) - running = asyncio.ensure_future(first.run_slot(slot, ctx)) - try: - # Past its first turn: the box holds work the trace recorded. - async with asyncio.timeout(600): - while not (slot.traces and slot.traces[0].num_turns >= 1): - assert not running.done(), "the rollout ended before the save" - await asyncio.sleep(0.2) - (point,) = await save_all() - finally: - running.cancel() - await asyncio.gather(running, return_exceptions=True) + slot, point = await saved_rollout(config, ctx) turns_saved = len([node for node in point.trace["nodes"] if node["sampled"]]) point = pickle.loads(pickle.dumps(point)) @@ -212,20 +222,7 @@ async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( client=config.client, model=config.model, sampling=config.sampling ) - first = load_environment(config.env) - (task,) = list(first.taskset) - async with first.serving(): - (slot,) = first.slots(task) - running = asyncio.ensure_future(first.run_slot(slot, ctx)) - try: - async with asyncio.timeout(600): - while not (slot.traces and slot.traces[0].num_turns >= 1): - assert not running.done(), "the rollout ended before the save" - await asyncio.sleep(0.2) - (point,) = await save_all() - finally: - running.cancel() - await asyncio.gather(running, return_exceptions=True) + slot, point = await saved_rollout(config, ctx) snapshot_id = point.snapshot["id"] point.snapshot["id"] = "no-such-checkpoint" @@ -244,3 +241,49 @@ async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( await asyncio.sleep(2) assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid([SUB_AGENTS], needs=NEEDS), indirect=True +) +async def test_another_owner_continues_running_sub_agents( + taken, harness, harness_runtime +): + """A rollout is saved while two sub-agents of its harness are at work (subagent- + restore). The second owner's box continues both: neither is started again, each + takes its next turns, and each reads its nonce from a process that only a box with + its memory still runs.""" + config = _eval_config( + "subagent-restore", + harness=harness, + runtime=harness_runtime, + max_turns=40, + max_tokens=4096, + rollout_timeout=600, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + slot, point = await saved_rollout( + config, ctx, ready=lambda trace: all(sub_agent_turns(trace).values()) + ) + saved = len(point.trace["nodes"]) + point = pickle.loads(pickle.dumps(point)) + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.has_restored and trace.attempt == 2 + assert trace.reward == 1.0 + for name, turns in sub_agent_turns(trace).items(): + # At work when the box was saved, and at work again in the restored box. + assert [turn for turn in turns if turn < saved], name + assert [turn for turn in turns if turn >= saved], name + # The coordinator and each sub-agent started once: three conversations. + assert sum(node.parent is None for node in trace.nodes) == 3 + (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] + await assert_no_leaks(taken) From 496124e369faf07629d8454240d9805eccab5e63 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 11 Nov 2026 21:25:04 +0000 Subject: [PATCH 40/53] install the relay before the harness setup starts The relay starts alongside the harness setup. Its install puts uv in a box that has none, and the setup of some harnesses (prime-agent, hermes-agent) does the same in a shell of its own. The two installs could overlap, and one then ran the other's half-written binary: "uv: Permission denied" from the relay start, seen once in CI. Co-Authored-By: Claude --- verifiers/v1/interception/relay/__init__.py | 22 +++++++++++++++++++-- verifiers/v1/rollout.py | 1 + 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index f8640de839..6dd2abf841 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -39,8 +39,9 @@ def relay_config(upstream: str, tokens: dict[str, str]) -> bytes: class Relay: """A rollout's relay in its box. It launches when created, before its upstream is - known, so its start overlaps the harness setup; `connect` points it at the - interception server, `reroute` moves it, `stop` ends it.""" + known, so its start overlaps the harness setup once its program is installed + (`installed`); `connect` points it at the interception server, `reroute` moves it, + `stop` ends it.""" def __init__( self, runtime: Runtime, name: str, *, running: dict[str, str] | None = None @@ -54,6 +55,9 @@ def __init__( self.upstream = "" self.tokens: dict[str, str] = {} self._connected = asyncio.Event() + self._installed = asyncio.Event() + if running is not None: + self._installed.set() self._launch = asyncio.create_task( self._start() if running is None else asyncio.sleep(0) ) @@ -86,6 +90,7 @@ async def _start(self) -> None: self.config_path, relay_config("http://127.0.0.1:9", {}) ) program = await self.runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + self._installed.set() args = ["--config", self.config_path, "--port", "0", "--ready", ready] args += ["--failures", self.failures_path] await self.runtime.run_background( @@ -103,6 +108,19 @@ async def _start(self) -> None: ) await asyncio.sleep(0.1) + async def installed(self) -> None: + """Wait until the relay's program is in the box, or its launch failed. The + install puts `uv` in the box when it has none, and a harness setup that does + the same must not run at the same time: one would run the other's half-written + binary.""" + waiter = asyncio.create_task(self._installed.wait()) + try: + await asyncio.wait( + {waiter, self._launch}, return_when=asyncio.FIRST_COMPLETED + ) + finally: + waiter.cancel() + async def connect(self, upstream: str, tokens: dict[str, str]) -> str: """Point the relay at the interception server once it runs, and return the base URL the harness reaches the server at.""" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index d09b3c5f4b..60b8751a4d 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -503,6 +503,7 @@ async def open(self) -> bool: running=self._restore.live["relay"] if restored else None, ) self._stack.push_async_callback(self._relay.stop) + await self._relay.installed() # A restored box still runs the harness. if not restored: await self.harness.setup(self._agent_runtime) From ae50ebe8523c947cf827132796ba51006285b073 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 11 Nov 2026 21:25:04 +0000 Subject: [PATCH 41/53] install the relay before the harness setup starts The relay starts alongside the harness setup. Its install puts uv in a box that has none, and the setup of some harnesses (prime-agent, hermes-agent) does the same in a shell of its own. The two installs could overlap, and one then ran the other's half-written binary: "uv: Permission denied" from the relay start, seen once in CI. Co-Authored-By: Claude --- verifiers/v1/interception/relay/__init__.py | 20 ++++++++++++++++++-- verifiers/v1/rollout.py | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py index 8ce2af2967..6aca99a876 100644 --- a/verifiers/v1/interception/relay/__init__.py +++ b/verifiers/v1/interception/relay/__init__.py @@ -39,8 +39,9 @@ def relay_config(upstream: str, tokens: dict[str, str]) -> bytes: class Relay: """A rollout's relay in its box. It launches when created, before its upstream is - known, so its start overlaps the harness setup; `connect` points it at the - interception server, `reroute` moves it, `stop` ends it.""" + known, so its start overlaps the harness setup once its program is installed + (`installed`); `connect` points it at the interception server, `reroute` moves it, + `stop` ends it.""" def __init__(self, runtime: Runtime, name: str) -> None: self.runtime = runtime @@ -50,6 +51,7 @@ def __init__(self, runtime: Runtime, name: str) -> None: self.upstream = "" self.tokens: dict[str, str] = {} self._connected = asyncio.Event() + self._installed = asyncio.Event() self._launch = asyncio.create_task(self._start()) @property @@ -74,6 +76,7 @@ async def _start(self) -> None: self.config_path, relay_config("http://127.0.0.1:9", {}) ) program = await self.runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + self._installed.set() args = ["--config", self.config_path, "--port", "0", "--ready", ready] args += ["--failures", self.failures_path] await self.runtime.run_background( @@ -91,6 +94,19 @@ async def _start(self) -> None: ) await asyncio.sleep(0.1) + async def installed(self) -> None: + """Wait until the relay's program is in the box, or its launch failed. The + install puts `uv` in the box when it has none, and a harness setup that does + the same must not run at the same time: one would run the other's half-written + binary.""" + waiter = asyncio.create_task(self._installed.wait()) + try: + await asyncio.wait( + {waiter, self._launch}, return_when=asyncio.FIRST_COMPLETED + ) + finally: + waiter.cancel() + async def connect(self, upstream: str, tokens: dict[str, str]) -> str: """Point the relay at the interception server once it runs, and return the base URL the harness reaches the server at.""" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 16346aaa78..fc7da38ba8 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -363,6 +363,7 @@ async def open(self) -> bool: if not runtime.is_local: self._relay = Relay(runtime, self.trace.id) self._stack.push_async_callback(self._relay.stop) + await self._relay.installed() await self.harness.setup(self._agent_runtime) # `base_url` is the interception server's reachable URL for this rollout. # The harness reaches the model at `{base_url}/v1`; tool servers reach this From 2e0c806e14700fd018fef2d9a82c49d4ee5aee79 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 11 Nov 2026 23:47:03 +0000 Subject: [PATCH 42/53] pace memory snapshots run-wide An owner that saves every running rollout pauses all their boxes at once. At thousands of rollouts that is a burst of pause and resume calls the platform may refuse, and a box whose resume fails never runs its rollout again. snapshots_per_min spreads the pauses out with the bucket that already paces creates. Off by default. Co-Authored-By: Claude --- verifiers/v1/configs/runtime.py | 4 ++++ verifiers/v1/runtimes/backends/prime.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 1cd5cf71f9..de784d0a36 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -223,6 +223,10 @@ class PrimeConfig(NetworkPolicyConfig): """Pace sandbox creation to this many per minute, enforced run-wide across every process of the run (None/<= 0 disables it). (Tunnel creation is limited separately — see interception.tunnel.prime.tunnel_limiter.)""" + snapshots_per_min: int | None = None + """Pace memory snapshots (a pause and a resume each) to this many per minute, + run-wide like ``creates_per_min`` (None/<= 0 disables it). An owner that saves + every running rollout at once (`save_all`) then spreads its pauses out.""" mounts: dict[str, PrimeVolume] = Field(default_factory=dict) """Guest paths mapped to Prime volumes, created if missing before the VM. diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 239d39b6b0..ca86312858 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -417,7 +417,15 @@ async def create() -> Snapshot: return snapshot try: - return await run_shielded(create()) + async with ( + creation_limiter( + (self.config.snapshots_per_min or 0) / 60, + "prime-snapshot", + run_scope(), + ) + or contextlib.nullcontext() + ): + return await run_shielded(create()) except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e From ad0eb22840ba8c3b0a01a1f76d819cf2f3026c86 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 11 Nov 2026 23:47:04 +0000 Subject: [PATCH 43/53] continue a saved rollout from its trace alone A saved rollout was a RestorePoint: the trace plus fields beside it that only that class carried. Whoever held just the trace could not continue it. save_all now returns the trace as plain data, with what a restore needs on the entry of its checkpoint, and a restore takes that trace. Snapshots on a failed model call are now opt-in (checkpoint.on_error). They help only when the harness would end on the error; a harness that handles the failed call itself never uses the snapshot, and a run without retries never takes one. Co-Authored-By: Claude --- docs/agent.md | 16 +-- tests/integration/test_checkpoint.py | 23 ++-- tests/unit/test_checkpoint.py | 21 ++-- verifiers/v1/agent.py | 23 ++-- verifiers/v1/configs/agent.py | 27 +++-- verifiers/v1/env.py | 7 +- verifiers/v1/restore.py | 168 ++++++++++++--------------- verifiers/v1/rollout.py | 24 ++-- verifiers/v1/trace.py | 4 + 9 files changed, 160 insertions(+), 153 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 344cb8ab10..f185f818ec 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,13 +77,14 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. +Rollout checkpointing is on by default: a rollout can be snapshotted, and a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. -A rollout takes a snapshot only when it needs one: +By default a rollout takes a snapshot only when its owner saves it (`save_all`, below). Two settings add snapshots that the rollout takes by itself: -- When a model call fails with an error that the run retries (`[env.agent.retries]`). The snapshot is taken before the harness gets the error, so the box that the retry restores still waits for the reply and asks for the call again. A rollout that never fails takes no snapshot and is never stopped. -- When its owner saves it (`save_all`, below). -- With `interval`, also at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. Without it, a retry after such a failure starts over. +- `on_error`: when a model call fails with an error that the run retries (`[env.agent.retries]`). The snapshot is taken before the harness gets the error, so the box that the retry restores still waits for the reply and asks for the call again. This helps only when the harness would end on the error. A harness that handles a failed call itself never uses the snapshot. +- `interval`: at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. + +Without a snapshot, a retry starts over. A snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. @@ -98,8 +99,9 @@ A retry also starts over when a restore fails before the restored attempt's firs ```toml [env.agent.checkpoint] -# interval = 10 # also snapshot at the 10th, 20th, ... model turn (off by default) +# on_error = true # snapshot when a model call fails with an error the run retries (off by default) +# interval = 10 # snapshot at the 10th, 20th, ... model turn (off by default) keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` -An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns a `RestorePoint` for each: plain data the owner stores. A later process continues a rollout from its point with `RunSlot(task, id=..., restore=point)` (or `Agent.run(task, restore=point)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. A restore point continues one rollout: an episode with more than one agent run cannot continue from the point of one of them, and starts over. +An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns the trace of each as plain data the owner stores. The trace holds everything a restore needs, on the entry of its checkpoint. A later process continues a rollout from its saved trace with `RunSlot(task, id=..., restore=trace)` (or `Agent.run(task, restore=trace)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. A saved trace continues one rollout: an episode with more than one agent run cannot continue from the trace of one of them, and starts over. diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index 4835b5403e..fc8db08015 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -20,7 +20,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.env import RunSlot -from verifiers.v1.restore import save_all +from verifiers.v1.restore import restore_entry, save_all from verifiers.v1.runtimes import PrimeBackend, Snapshot from verifiers.v1.utils.loaders import harness_class, load_environment @@ -73,7 +73,12 @@ async def restored_trace( harness=harness, runtime=harness_runtime, taskset_overrides={"task": {"fail_turn": fail_turn}}, - env={"agent": {"retries": RETRIES, "checkpoint": checkpoint}}, + env={ + "agent": { + "retries": RETRIES, + "checkpoint": {"on_error": True, **checkpoint}, + } + }, max_turns=12, max_tokens=4096, ) @@ -175,7 +180,7 @@ async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_r ) slot, point = await saved_rollout(config, ctx) - turns_saved = len([node for node in point.trace["nodes"] if node["sampled"]]) + turns_saved = len([node for node in point["nodes"] if node["sampled"]]) point = pickle.loads(pickle.dumps(point)) second = load_environment(config.env) @@ -184,7 +189,7 @@ async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_r episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) (trace,) = episode.traces assert trace.ok, [error.message for error in trace.errors] - assert trace.id == point.id + assert trace.id == point["id"] assert trace.reward == 1.0 assert trace.metrics["copied"] == 1.0, [ (n.message.role, n.sampled, str(n.message.model_dump(exclude_none=True))[:600]) @@ -195,7 +200,7 @@ async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_r assert sum(node.parent is None for node in trace.nodes) == 1 # The owner holds the snapshot: no rollout deleted it. (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] - assert snapshot_id == point.checkpoint + assert snapshot_id == restore_entry(point)["id"] async with AsyncSandboxClient() as client: assert (await client.get_checkpoint(snapshot_id)).state == "DURABLE" assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] @@ -223,8 +228,8 @@ async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( ) slot, point = await saved_rollout(config, ctx) - snapshot_id = point.snapshot["id"] - point.snapshot["id"] = "no-such-checkpoint" + snapshot = restore_entry(point)["snapshot"] + snapshot_id, snapshot["id"] = snapshot["id"], "no-such-checkpoint" second = load_environment(config.env) (task,) = list(second.taskset) @@ -232,7 +237,7 @@ async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) (trace,) = episode.traces assert trace.ok, [error.message for error in trace.errors] - assert trace.id == point.id + assert trace.id == point["id"] assert trace.reward == 1.0 assert trace.attempt == 3 @@ -267,7 +272,7 @@ async def test_another_owner_continues_running_sub_agents( slot, point = await saved_rollout( config, ctx, ready=lambda trace: all(sub_agent_turns(trace).values()) ) - saved = len(point.trace["nodes"]) + saved = len(point["nodes"]) point = pickle.loads(pickle.dumps(point)) second = load_environment(config.env) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 03ec373c72..26a649ec06 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -14,7 +14,7 @@ from verifiers.v1.configs.client import EvalClientConfig from verifiers.v1.configs.retries import RetryConfig from verifiers.v1.graph import MessageNode -from verifiers.v1.restore import RestorePoint, unsaveable +from verifiers.v1.restore import open_restore, restore_entry, unsaveable from verifiers.v1.rollout import Rollout, kept from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import ( @@ -348,6 +348,7 @@ def test_retry_state_tells_whether_an_error_gets_a_retry_without_spending_it(): def test_checkpointing_is_on_by_default(): assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() assert RolloutCheckpointConfig().interval is None + assert not RolloutCheckpointConfig().on_error assert gated("bash", vf.PrimeConfig())._checkpointing() is not None @@ -423,7 +424,7 @@ async def failing(): assert session.flowing.is_set() and run.trace.checkpoints == [] -async def test_save_returns_a_restore_point_another_process_can_open(): +async def test_save_returns_a_trace_another_process_can_continue(): run, backend, session = make_rollout() session.pinned_tool_results = { ("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned") @@ -431,16 +432,20 @@ async def test_save_returns_a_restore_point_another_process_can_open(): run.trace.info = {"nonce": "n"} backend.running.set() backend.durable.set() - point = await run.save() - assert isinstance(point, RestorePoint) + saved = await run.save() + assert saved is not None (checkpoint,) = run.trace.checkpoints assert checkpoint.pinned + # The running rollout's own trace does not carry the saved fields. + assert checkpoint.restore is None + assert "restore" not in run.trace.model_dump()["checkpoints"][0] - point = pickle.loads(pickle.dumps(point)) - assert point.id == run.trace.id - assert point.snapshot == {"id": "ck1"} + saved = pickle.loads(pickle.dumps(saved)) + assert saved["id"] == run.trace.id + assert restore_entry(saved)["snapshot"] == {"id": "ck1"} task = vf.Task(vf.TaskData(idx=0, prompt="q")) - trace, restored = point.open(task) + trace, restored = open_restore(saved, task) + assert restored.restore is None assert trace.id == run.trace.id and trace.num_turns == run.trace.num_turns assert restored is trace.checkpoints[0] and restored.pinned assert restored.info == {"nonce": "n"} diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 8636efe78b..0bc11c090d 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -35,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.restore import RestorePoint, restoring, unsaveable +from verifiers.v1.restore import open_restore, restoring, unsaveable from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, @@ -711,7 +711,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, - restore: RestorePoint | None = None, + restore: dict[str, Any] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -730,7 +730,7 @@ async def run( the selected verifier workspace, inside its timeout and retry boundary. `restore` continues a rollout another process saved (`save_all`) instead of starting one: its trace, in a new box from its memory checkpoint; the episode's - restore point (`RunSlot.restore`) is taken the same way.""" + saved trace (`RunSlot.restore`) is taken the same way.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries @@ -741,6 +741,7 @@ async def run( trace: Trace | None = None target: Checkpoint | None = None # the checkpoint the next attempt restores first_attempt = 0 # the attempts the trace made before this run + offered: str | None = None # the checkpoint of the owner's saved trace pending = restoring.get() if restore is None and pending: restore = pending.pop() @@ -748,11 +749,12 @@ async def run( if checkpointing is None: logger.warning( "Starting rollout %s over: this agent takes no checkpoints, so it " - "cannot continue from its restore point", - restore.id, + "cannot continue from its saved trace", + restore.get("id"), ) else: - trace, target = restore.open(task) + trace, target = open_restore(restore, task) + offered = target.id trace_id = trace.id first_attempt = trace.attempt if on_trace is not None: @@ -824,16 +826,15 @@ async def run( if trace.ok: break if ( - restore is not None - and target is not None - and target.id == restore.checkpoint + target is not None + and target.id == offered and target.id in failed_restores ): - # The owner's restore point is an offer, not an attempt of + # The owner's saved trace is an offer, not an attempt of # this run: without it the rollout would have started over, # so it does that now and keeps its retries. logger.warning( - "Starting rollout over: restore point %s led nowhere - %s", + "Starting rollout over: saved trace %s led nowhere - %s", target.id, clip(trace.errors[-1].message) if trace.errors else "", ) diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 9575922fc7..34eb5858fe 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -40,19 +40,26 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing: snapshot the box's files and memory when a model call - fails with an error the run retries, so the retry continues from there instead of - starting over. The snapshot is taken before the harness gets the error: the - restored box continues the harness program itself and asks for the call again. A - rollout that never fails takes no snapshot. A snapshot stops the box for a few - seconds. It needs a runtime that takes memory snapshots + """Rollout checkpointing: snapshot the box's files and memory, so a retry or + another process continues the rollout from there instead of starting over. The + restored box continues the harness program itself and asks again for the model + call that was open. By default a rollout takes a snapshot only when its owner + saves it (`save_all`); `on_error` and `interval` add snapshots the rollout takes + by itself. A snapshot stops the box for a few seconds. It needs a runtime that + takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services of its own. Elsewhere a retry starts over. Every snapshot is deleted once the run is final.""" + on_error: bool = False + """Snapshot when a model call fails with an error the run retries, before the + harness gets the error: the retry continues the box that still waits for the + reply. It helps when the harness would end on the error; a harness that handles a + failed call itself (its SDK retries, or a sub-agent is started again) never uses + the snapshot.""" interval: PositiveInt | None = None - """Also snapshot at every n-th model turn (the n-th, the 2n-th, ...), for failures + """Snapshot at every n-th model turn (the n-th, the 2n-th, ...), for failures that leave no box to snapshot (the box died, the harness program crashed): a retry then loses fewer than n turns instead of starting over. None takes none.""" keep: Literal["last"] | PositiveInt = "last" @@ -101,9 +108,9 @@ class AgentConfig(BaseConfig): checkpoint: RolloutCheckpointConfig | None = Field( default_factory=RolloutCheckpointConfig ) - """Rollout checkpointing, on by default: snapshot the box as turns go upstream, so - a retry restores the newest snapshot instead of starting over (see - `RolloutCheckpointConfig`). None turns it off.""" + """Rollout checkpointing, on by default: the rollout can be snapshotted, so its + owner can save it and a retry restores its newest snapshot instead of starting + over (see `RolloutCheckpointConfig`). None turns it off.""" network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index a51dbb44de..11d7591d81 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import ( + Any, Generic, TypeVar, ) @@ -30,7 +31,7 @@ requires_tunnel, ) from verifiers.v1.mcp import SharedToolServer, serve_shared -from verifiers.v1.restore import RestorePoint, restoring +from verifiers.v1.restore import restoring from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task from verifiers.v1.trace import Error, Trace, TraceTask @@ -68,7 +69,7 @@ class RunSlot: episode: Episode | None = None done: bool = False started: float | None = None - restore: RestorePoint | None = None + restore: dict[str, Any] | None = None """A rollout another process saved (`save_all`): the episode's first attempt continues it instead of starting one.""" @@ -250,7 +251,7 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, - restore: RestorePoint | None = None, + restore: dict[str, Any] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py index 41fab1a30f..f54bb4130e 100644 --- a/verifiers/v1/restore.py +++ b/verifiers/v1/restore.py @@ -1,15 +1,14 @@ -"""Restore points: a running rollout's memory checkpoint in a form another process can -continue. +"""Saving running rollouts, so another process can continue them. A retry restores a checkpoint in the process that took it, from the live `Checkpoint`. An owner that outlives its process (a training run that stops and resumes) saves every running rollout instead (`save_all`): each one takes a memory checkpoint now and -returns a `RestorePoint`, made of plain data the owner can store. A later process hands -the point back (`RunSlot.restore`, `Agent.run(restore=)`), and the rollout continues in -a new box from the checkpoint: its trace up to there, and the harness program the box -still runs. +returns its trace as plain data (`restorable`), which holds everything a restore +needs. A later process hands that trace back (`RunSlot.restore`, `Agent.run(restore=)`), +and the rollout continues in a new box from the checkpoint: its trace up to there, and +the harness program the box still runs. -The owner holds the snapshot of a restore point (`Checkpoint.pinned`): no rollout +The owner holds the snapshot of a saved rollout (`Checkpoint.pinned`): no rollout deletes it, and the owner deletes it once nothing will restore it (`RuntimeBackend.discard`). """ @@ -19,7 +18,6 @@ import asyncio import weakref from contextvars import ContextVar -from dataclasses import dataclass from typing import TYPE_CHECKING, Any from pydantic import TypeAdapter @@ -41,89 +39,71 @@ LIVE: weakref.WeakSet[Rollout] = weakref.WeakSet() """The rollouts of this process that run a box (between `open` and `close`).""" -restoring: ContextVar[list[RestorePoint] | None] = ContextVar( +restoring: ContextVar[list[dict[str, Any]] | None] = ContextVar( "vf_restoring", default=None ) -"""The restore point the episode this task runs continues from (`Env.run_episode`), in -a list its first agent run takes it out of.""" - - -@dataclass -class RestorePoint: - """A rollout as a memory checkpoint left it, as plain data: every field survives a - pickle or a msgpack round trip with raw bytes and numpy arrays.""" - - trace: dict[str, Any] - """The trace when the checkpoint was taken, with its token arrays - (`Trace.model_dump(context={"token_arrays": True})`).""" - checkpoint: str - """The id of the checkpoint on `trace` to continue from.""" - state: Any - """The trace's typed state, as JSON data.""" - info: dict[str, Any] - pins: list[tuple[str, str, dict[str, Any]]] - """The pinned tool results: conversation prefix, call id and the message.""" - replays: dict[str, tuple[str, bytes, int, bytes, str]] - """The replies the box may ask for again: request path and digest, then the reply's - status, body and content type.""" - live: dict[str, Any] - """`Checkpoint.live`.""" - - @property - def id(self) -> str: - """The trace's id.""" - return self.trace["id"] - - @property - def snapshot(self) -> dict[str, Any]: - """The box snapshot the point continues from (`Snapshot` as data).""" - for checkpoint in self.trace["checkpoints"]: - if checkpoint["id"] == self.checkpoint: - return checkpoint["snapshot"] - raise ValueError(f"restore point {self.id} lost its checkpoint") - - @classmethod - def of(cls, trace: Trace, checkpoint: Checkpoint) -> RestorePoint: - return cls( - trace=trace.model_dump(context={"token_arrays": True}), - checkpoint=checkpoint.id, - state=TypeAdapter(type(checkpoint.state)).dump_python( - checkpoint.state, mode="json" - ), - info=dict(checkpoint.info or {}), - pins=[ - (prefix, call_id, message.model_dump(mode="json")) - for (prefix, call_id), message in checkpoint.pins.items() - ], - replays={ - key: (path, digest, reply.status, reply.body, reply.content_type) - for key, ((path, digest), reply) in checkpoint.replays.items() - }, - live=checkpoint.live, - ) - - def open(self, task: Task) -> tuple[Trace, Checkpoint]: - """The trace and its live checkpoint, for `task`'s rollout to restore.""" - trace = Trace.model_validate(self.trace) - checkpoint = next(c for c in trace.checkpoints if c.id == self.checkpoint) - trace.state = state_cls(type(task)).model_validate(self.state) - checkpoint.state = trace.state.model_copy(deep=True) - checkpoint.info = dict(self.info) - checkpoint.pins = { - (prefix, call_id): ToolMessage.model_validate(message) - for prefix, call_id, message in self.pins - } - checkpoint.replays = { - key: ((path, digest), ReplayResponse(status, body, content_type)) - for key, (path, digest, status, body, content_type) in self.replays.items() - } - checkpoint.live = self.live - return trace, checkpoint +"""The saved trace the episode this task runs continues from (`Env.run_episode`), in a +list its first agent run takes it out of.""" + + +def restorable(trace: Trace, checkpoint: Checkpoint) -> dict[str, Any]: + """`trace` as plain data that any process can continue from `checkpoint`: its dump + with token arrays, and on the checkpoint's entry what the checkpoint holds only + while its rollout lives (`Checkpoint.restore`). Every value survives a pickle or a + msgpack round trip with raw bytes and numpy arrays.""" + data = trace.model_dump(context={"token_arrays": True}) + entry = next(c for c in data["checkpoints"] if c["id"] == checkpoint.id) + entry["restore"] = { + "state": TypeAdapter(type(checkpoint.state)).dump_python( + checkpoint.state, mode="json" + ), + "info": dict(checkpoint.info or {}), + "pins": [ + (prefix, call_id, message.model_dump(mode="json")) + for (prefix, call_id), message in checkpoint.pins.items() + ], + "replays": { + key: (path, digest, reply.status, reply.body, reply.content_type) + for key, ((path, digest), reply) in checkpoint.replays.items() + }, + "live": checkpoint.live, + } + return data + + +def restore_entry(trace: dict[str, Any]) -> dict[str, Any]: + """The checkpoint entry a saved trace continues from: the newest one that holds + what a restore needs.""" + for entry in reversed(trace.get("checkpoints") or []): + if entry.get("restore") is not None: + return entry + raise ValueError(f"trace {trace.get('id')} holds no checkpoint to continue from") + + +def open_restore(data: dict[str, Any], task: Task) -> tuple[Trace, Checkpoint]: + """The trace and its live checkpoint, for `task`'s rollout to restore.""" + saved = restore_entry(data) + trace = Trace.model_validate(data) + checkpoint = next(c for c in trace.checkpoints if c.id == saved["id"]) + held, checkpoint.restore = saved["restore"], None + trace.state = state_cls(type(task)).model_validate(held["state"]) + checkpoint.state = trace.state.model_copy(deep=True) + checkpoint.info = dict(held["info"]) + checkpoint.pins = { + (prefix, call_id): ToolMessage.model_validate(message) + for prefix, call_id, message in held["pins"] + } + checkpoint.replays = { + key: ((path, digest), ReplayResponse(status, body, content_type)) + for key, (path, digest, status, body, content_type) in held["replays"].items() + } + checkpoint.live = held["live"] + return trace, checkpoint def unsaveable(config: AgentConfig, harness: Harness | None = None) -> str | None: """Why the rollouts of an agent with `config` take no memory snapshots, so `save_all` - returns no restore point for them; None when they take them. A task with tool + returns nothing for them; None when they take them. A task with tool servers or services of its own rules its rollout out as well, which only the rollout knows.""" from verifiers.v1.runtimes import backend_cls @@ -153,16 +133,16 @@ def release_all() -> None: async def save_all( skipped: dict[str, int] | None = None, *, hold: bool = False -) -> list[RestorePoint]: +) -> list[dict[str, Any]]: """Take a memory checkpoint of every running rollout of this process and return - their restore points. A rollout that takes no memory snapshots, runs no harness - segment right now, or whose snapshot fails has none: it starts over when its owner - runs it again. `skipped` collects why, as a count per reason. The snapshots are - pinned: the caller owns them. With `hold`, each saved rollout waits at its restore - point until `release_all`, so the caller can record which rollouts are still - running without one of them finishing meanwhile.""" - - async def save(rollout: Rollout) -> RestorePoint | None: + their traces as saved (`restorable`). A rollout that takes no memory snapshots, + runs no harness segment right now, or whose snapshot fails has none: it starts + over when its owner runs it again. `skipped` collects why, as a count per reason. + The snapshots are pinned: the caller owns them. With `hold`, each saved rollout + waits at its checkpoint until `release_all`, so the caller can record which + rollouts are still running without one of them finishing meanwhile.""" + + async def save(rollout: Rollout) -> dict[str, Any] | None: try: point = await rollout.save(hold=hold) reason = rollout.save_skipped diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 60b8751a4d..92d7ce0ac3 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal from urllib.parse import urlsplit from verifiers.v1.clients import ModelContext @@ -36,7 +36,7 @@ from verifiers.v1.interception import Interception, serve_interception from verifiers.v1.interception.relay import Relay from verifiers.v1.mcp import SharedToolServer, serve_tools -from verifiers.v1.restore import LIVE, RestorePoint, unsaveable +from verifiers.v1.restore import LIVE, restorable, unsaveable from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -529,7 +529,8 @@ async def open(self) -> bool: and self._memory ): self._session.checkpointer = self._checkpoint - self._session.rescuer = self._rescue + if self._checkpointing.on_error: + self._session.rescuer = self._rescue logger.debug( "Set up %s harness in %s", self.harness.config.name, @@ -961,13 +962,14 @@ async def _rescue(self, error: Exception) -> None: def _last_node(self) -> int | None: return len(self.trace.nodes) - 1 if self.trace.nodes else None - async def save(self, *, hold: bool = False) -> RestorePoint | None: - """Take a memory checkpoint of the running rollout now and return its restore - point, for an owner that continues the rollout in another process - (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`); it becomes - restorable from any node in the background, and this run's teardown waits for - that. With `hold`, nothing the box asked for is answered until `release`, so - the rollout cannot move past its restore point while the owner records it. + async def save(self, *, hold: bool = False) -> dict[str, Any] | None: + """Take a memory checkpoint of the running rollout now and return its trace as + saved (`restorable`), for an owner that continues the rollout in another + process (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`); it + becomes restorable from any node in the background, and this run's teardown + waits for that. With `hold`, nothing the box asked for is answered until + `release`, so the rollout cannot move past its checkpoint while the owner + records it. None when the rollout takes no memory snapshots (`unsaveable`), runs no segment, or the snapshot failed; `save_skipped` then says which.""" self.save_skipped = None @@ -998,7 +1000,7 @@ async def save(self, *, hold: bool = False) -> RestorePoint | None: self._release_timer = asyncio.get_running_loop().call_later( HOLD_SECONDS, self.release ) - return RestorePoint.of(self.trace, checkpoint) + return restorable(self.trace, checkpoint) def release(self) -> None: """Answer the box again after a `save(hold=True)`.""" diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 75aff17e79..dc919d6ed6 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -404,6 +404,10 @@ class Checkpoint(BaseModel): """The replies the server had just sent when the snapshot stopped the box (`RolloutSession.idempotent_requests`): the restored box asks again for one that had not arrived, and gets the reply the trace already holds. Live-only.""" + restore: dict[str, Any] | None = Field(None, exclude_if=lambda value: value is None) + """The live-only fields above as plain data, on a trace saved for another process + to continue (`verifiers.v1.restore.restorable`); None on a running rollout's + checkpoint, which holds them live.""" time: TimeSpan = Field(default_factory=TimeSpan) """From the snapshot request until it was durable (restorable); `end` stays 0 while it is not.""" From c16809f7f5356d2449d01f3efa4815284898fea5 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 00:51:44 +0000 Subject: [PATCH 44/53] run the harness program in a session on prime A box that boots from a memory snapshot keeps the sandboxd session of the harness program, so the host can attach to it by its session id. The program no longer needs a polled background job, and it gets the same protection as every other command against a task that clears the scratch directories. A snapshot also ends the stream into the box that was paused. The wait now attaches to the session again when the box runs. Needs a prime-sandboxes release with attach_process. Co-Authored-By: Claude --- verifiers/v1/runtimes/backends/prime.py | 136 ++++++++---------------- verifiers/v1/runtimes/base.py | 10 +- 2 files changed, 50 insertions(+), 96 deletions(-) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index ab5088adc0..c69e2512a8 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -45,16 +45,6 @@ EFFECTIVELY_UNBOUNDED_SECONDS = 30 * 24 * 60 * 60 """Safety deadline for APIs that require a finite bound. Normal execution remains bounded by idle detection or rollout cancellation; 30 days is above any real run.""" -# The SDK reads finished jobs' output through a bounded pool whose deadline includes time -# spent queued; its defaults (20 reads, 45s) expire most reads when thousands of rollouts -# finish a job at once (e.g. every harness program of a cancelled step). -_OUTPUT_READS = 100 -"""Concurrent output reads on the shared client.""" -_OUTPUT_DEADLINE_SECONDS = 300 -"""Deadline for one read of a finished job's output, queue time included.""" -_OUTPUT_RETRIES = 10 -"""Re-reads of a finished job's output that the SDK still failed to fetch, before the -exec is reported as failed.""" _CHECKPOINT_DURABLE_SECONDS = 300 _PAUSE_SETTLE_SECONDS = 120 """How long a pause or a resume gets to take effect; they take seconds.""" @@ -148,9 +138,7 @@ def _lease_client() -> Any: loop = asyncio.get_running_loop() shared = _shared_clients.get(loop) if shared is None: - shared = _shared_clients[loop] = _SharedClient( - AsyncSandboxClient(background_job_output_concurrency=_OUTPUT_READS) - ) + shared = _shared_clients[loop] = _SharedClient(AsyncSandboxClient()) shared.leases += 1 return shared.client @@ -240,6 +228,8 @@ def __init__(self, config: PrimeConfig, name: str | None = None) -> None: """Cleared while a memory snapshot stops the box: the platform refuses every call into a stopped box, so calls wait for it.""" self._awake.set() + self._snapshots = 0 + """Memory snapshots taken of this box. Each one ended the open streams into it.""" @property def published_port(self) -> int: @@ -404,6 +394,7 @@ async def snapshot(self) -> Snapshot: async def create() -> Snapshot: # Shielded as one: a box left paused never runs its rollout again. self._awake.clear() + self._snapshots += 1 try: paused = await self._client.pause(self.info.id) snapshot = Snapshot(id=paused.checkpoint_id) @@ -588,16 +579,6 @@ async def run( program: Program | None = None, ) -> ProgramResult: await self._awake.wait() - if program is not None: - return await self._run_job(runtime, argv, env, program) - - async def collect(stream: AsyncIterator[bytes]) -> bytes: - tail = bytearray() - async for chunk in stream: - tail.extend(chunk[-_OUTPUT_TAIL_BYTES:]) - del tail[:-_OUTPUT_TAIL_BYTES] - return bytes(tail) - try: # Command sessions survive task cleanup of shared scratch directories. # Close stdin explicitly: the live-process API keeps it open by default. @@ -609,95 +590,68 @@ async def collect(stream: AsyncIterator[bytes]) -> bytes: env=runtime.process_env(env), user=runtime.user, ) - try: - stdout, stderr, exit_code = await asyncio.gather( - collect(process.stdout), collect(process.stderr), process.wait() - ) - finally: - # Closing signals and reaps a running process, then releases its stream. - await run_shielded(process.aclose()) - except ( - Exception - ) as e: # a sandbox/API failure is one rollout's problem, not the eval's - raise SandboxError(f"prime exec failed: {e}") from e - return ProgramResult( - exit_code=exit_code, - stdout=stdout.decode(errors="replace"), - stderr=stderr.decode(errors="replace"), - ) - - async def _run_job( - self, runtime: Runtime, argv: list[str], env: dict[str, str], program: Program - ) -> ProgramResult: - """Run the harness's main program as a server-owned job. A native session ends - with its box, and a box that boots from a memory snapshot is a new one; the job's - files and process come with the snapshot, so `attach` continues the wait.""" - try: - # Poll directly so rollout cancellation owns the execution timeout. - job = await self._client.start_background_job( - self.info.id, - shlex.join(argv), - working_dir=self.config.workdir, - env=runtime.process_env(env), - user=runtime.user, - ) except ( Exception ) as e: # a sandbox/API failure is one rollout's problem, not the eval's raise SandboxError(f"prime exec failed: {e}") from e - program.job = job.model_dump() + if program is not None: + program.job = {"session": process.session_uuid} try: - return await self._wait_job(job) + return await self._collect(process) finally: - program.job = None + if program is not None: + program.job = None async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: - from prime_sandboxes import BackgroundJob + try: + # The session and its process came with the snapshot; only the box is new. + process = await self._client.attach_process(self.info.id, job["session"]) + except Exception as e: + raise SandboxError(f"prime attach failed: {e}") from e + return await self._collect(process) - # The job's files and process came with the snapshot; only the box is new. - return await self._wait_job( - BackgroundJob(**{**job, "sandbox_id": self.info.id}) - ) + async def _collect(self, process: Any) -> ProgramResult: + """Wait for `process` and keep the tail of each output stream. A memory + snapshot stops the box, which ends the stream but not the process: attach to + it again when the box runs. The output of that gap can be missing.""" + + async def collect(stream: AsyncIterator[bytes], tail: bytearray) -> None: + async for chunk in stream: + tail.extend(chunk[-_OUTPUT_TAIL_BYTES:]) + del tail[:-_OUTPUT_TAIL_BYTES] - async def _wait_job(self, job: Any) -> ProgramResult: + stdout, stderr = bytearray(), bytearray() try: - delay = 0.1 - output_retries = 0 - missing = None while True: - await self._awake.wait() + snapshots = self._snapshots try: - result = await self._client.get_background_job( - self.info.id, job, timeout=_OUTPUT_DEADLINE_SECONDS + *_, exit_code = await asyncio.gather( + collect(process.stdout, stdout), + collect(process.stderr, stderr), + process.wait(), ) + break except Exception as e: - if not self._stopped_by_snapshot(e): + if snapshots == self._snapshots and not self._stopped_by_snapshot( + e + ): raise - continue - # Under load the SDK can see a job finish yet miss its output (its bounded - # output reads expire while queued) and reports that as `*_error`. The job - # is done, so asking again re-reads the output. - missing = result.stdout_error or result.stderr_error - if result.completed and ( - missing is None or output_retries == _OUTPUT_RETRIES - ): - break - if result.completed: - output_retries += 1 - await asyncio.sleep(delay) - delay = min(delay * 2, 3) + session = process.session_uuid + # A close would signal the process; a detach only drops the stream. + await run_shielded(process.detach()) + await self._awake.wait() + process = await self._client.attach_process(self.info.id, session) except ( Exception ) as e: # a sandbox/API failure is one rollout's problem, not the eval's raise SandboxError(f"prime exec failed: {e}") from e - if missing is not None: - raise SandboxError(f"prime exec output unavailable: {missing}") - if result.exit_code is None: - raise SandboxError("prime exec completed without an exit code") + finally: + # Closing signals and reaps a running process, then releases its stream. + await run_shielded(process.aclose()) return ProgramResult( - exit_code=result.exit_code, - stdout=result.stdout or "", - stderr=result.stderr or "", + exit_code=exit_code, + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), ) async def exec( diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index a695a0054b..7c0003d568 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -155,12 +155,12 @@ class Snapshot(BaseModel): @dataclass class Program: - """The background job of a harness's main program, as its runtime started it + """A harness's main program, as its runtime started it (`Runtime.run(..., program=)`). A box that boots from a memory snapshot still runs - the job, and `Runtime.attach` waits for it there.""" + it, and `Runtime.attach` waits for it there.""" job: dict[str, Any] | None = None - """The provider's handle of the job while it runs; None before and after.""" + """The provider's handle of the program while it runs; None before and after.""" current_program: ContextVar[Program | None] = ContextVar( @@ -269,8 +269,8 @@ async def snapshot(self) -> Snapshot: raise NotImplementedError(f"{type(self).__name__} does not support snapshots") async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: - """Wait for a job that a box started before the memory snapshot this box booted - from (`Program.job`), and return its result like `run`.""" + """Wait for a program that a box started before the memory snapshot this box + booted from (`Program.job`), and return its result like `run`.""" raise NotImplementedError( f"{type(self).__name__} does not support memory snapshots" ) From a1dd2ed2162be57ebaebfcf9b1690354d2706385 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 01:49:26 +0000 Subject: [PATCH 45/53] remove the snapshot on a failed model call A memory snapshot resets the box's open connections, so the relay asks again for the call that just failed. The interception server keeps no failed attempt, so the box gets a new model call and the harness rarely gets the error at all. The snapshot was a slow way to retry a model call, and its outcome depended on when the relay's request arrived. A retry of a model call belongs at the call. Snapshots stay for a box or a host that goes away: the owner's save, and `interval`. A call that the box asks again at a snapshotted turn now takes no second snapshot, or a call that keeps failing would take one per request. Co-Authored-By: Claude --- docs/agent.md | 8 +-- .../integration/fixtures/memory_restore_v1.py | 17 +++--- tests/integration/test_checkpoint.py | 29 +++------- tests/unit/test_checkpoint.py | 51 +++------------- tests/unit/test_retry_ids.py | 2 - verifiers/v1/agent.py | 9 +-- verifiers/v1/configs/agent.py | 12 +--- verifiers/v1/interception/server.py | 16 ----- verifiers/v1/rollout.py | 31 ++-------- verifiers/v1/session.py | 4 -- verifiers/v1/utils/retries.py | 58 +++++++------------ 11 files changed, 55 insertions(+), 182 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index f185f818ec..ad3d6b73ee 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -79,10 +79,7 @@ skip_on_timeout = true Rollout checkpointing is on by default: a rollout can be snapshotted, and a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. -By default a rollout takes a snapshot only when its owner saves it (`save_all`, below). Two settings add snapshots that the rollout takes by itself: - -- `on_error`: when a model call fails with an error that the run retries (`[env.agent.retries]`). The snapshot is taken before the harness gets the error, so the box that the retry restores still waits for the reply and asks for the call again. This helps only when the harness would end on the error. A harness that handles a failed call itself never uses the snapshot. -- `interval`: at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. +By default a rollout takes a snapshot only when its owner saves it (`save_all`, below). `interval` adds snapshots that the rollout takes by itself, at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. Without a snapshot, a retry starts over. @@ -95,11 +92,10 @@ A rollout takes snapshots only when all of these hold. Otherwise a retry starts - The rollout runs in a box of its own, not in a borrowed one. - The rollout has no tool servers, no shared tools and no services of its own. They reach the host at an address that does not follow the box to its restore. -A retry also starts over when a restore fails before the restored attempt's first new turn, unless a model call failed again. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +A retry also starts over when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] -# on_error = true # snapshot when a model call fails with an error the run retries (off by default) # interval = 10 # snapshot at the 10th, 20th, ... model turn (off by default) keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` diff --git a/tests/integration/fixtures/memory_restore_v1.py b/tests/integration/fixtures/memory_restore_v1.py index 69149b1033..76ce65f65b 100644 --- a/tests/integration/fixtures/memory_restore_v1.py +++ b/tests/integration/fixtures/memory_restore_v1.py @@ -7,10 +7,12 @@ The reward reads the nonce from the process. The task asks a harness with a shell for two commands that build on each other, and a metric checks the file they leave. -A response `@intercept` raises a `ProviderError` once, at the model turn `fail_turn`: -the harness gets the error as its reply and dies, after the snapshot of that turn -exists. The restored box still waits for that turn's reply, gets it from the retry, and -finishes: no new harness process, no notice to the model. +A response `@intercept` raises a `ProviderError` at the model turn `fail_turn`, for +every call of the first attempt: the harness gets the error as its reply and dies, +after the snapshot of that turn exists. The restored box still waits for that turn's +reply, gets it from the retry, and finishes: no new harness process, no notice to the +model. One failed call is not enough: the snapshot resets the box's connections, its +relay asks again, and a call that then succeeds would let the harness continue. """ import uuid @@ -40,10 +42,6 @@ class MemoryRestoreConfig(vf.TasksetConfig): class MemoryRestoreTask(vf.Task[vf.TaskData, vf.State, MemoryRestoreTaskConfig]): - failed = False - """Whether the injected failure fired: once per task, so a retry from the start - (checkpoints off) also gets through.""" - async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: nonce = uuid.uuid4().hex trace.info["nonce"] = nonce @@ -53,8 +51,7 @@ async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: @vf.intercept def fail_once(self, response: vf.Response, trace: vf.Trace) -> None: - if trace.num_turns + 1 >= self.config.fail_turn and not self.failed: - self.failed = True + if trace.attempt == 1 and trace.num_turns + 1 >= self.config.fail_turn: # A 4xx: harness SDKs end the turn at once instead of retrying it until the # rollout's deadline. raise vf.ProviderError("injected mid-rollout failure", status_code=400) diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index fc8db08015..100250e296 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -76,7 +76,7 @@ async def restored_trace( env={ "agent": { "retries": RETRIES, - "checkpoint": {"on_error": True, **checkpoint}, + "checkpoint": checkpoint, } }, max_turns=12, @@ -130,31 +130,20 @@ async def saved_rollout(config, ctx, ready=lambda trace: trace.num_turns >= 1): @pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): - """The one snapshot of the run is taken when the model call fails. A harness with - a shell fails at its second turn, so the restored box holds the first turn's - command and its harness goes on to the second; a harness without one has a single - turn, which fails.""" + """The box is snapshotted at every model turn, and the retry restores the snapshot + of the turn whose call failed. A harness with a shell fails at its second turn, so + the restored box holds the first turn's command and its harness goes on to the + second; a harness without one has a single turn, which fails. The calls that fail + again at that turn take no further snapshot.""" shell = harness_class(harness.id).EXECUTES_CODE + fail_turn = 2 if shell else 1 trace = await restored_trace( - run, harness, harness_runtime, fail_turn=2 if shell else 1 + run, harness, harness_runtime, fail_turn=fail_turn, interval=1 ) if shell: assert trace.num_turns >= 2 assert trace.metrics["copied"] == 1.0 - assert len(taken) == 1 - await assert_no_leaks(taken) - - -@pytest.mark.parametrize( - "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True -) -async def test_snapshots_at_turns_are_restored_too( - run, taken, harness, harness_runtime -): - """With `interval`, the box is also snapshotted at model turns. The failed call - takes no second snapshot of the same turn: the retry restores the one of turn 2.""" - trace = await restored_trace(run, harness, harness_runtime, fail_turn=2, interval=1) - assert [c.id for c in trace.checkpoints if c.restored] == [taken[1]] + assert [c.id for c in trace.checkpoints if c.restored] == [taken[fail_turn - 1]] await assert_no_leaks(taken) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 26a649ec06..fb02187174 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -12,7 +12,6 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.configs.client import EvalClientConfig -from verifiers.v1.configs.retries import RetryConfig from verifiers.v1.graph import MessageNode from verifiers.v1.restore import open_restore, restore_entry, unsaveable from verifiers.v1.rollout import Rollout, kept @@ -29,7 +28,6 @@ ToolMessage, UserMessage, ) -from verifiers.v1.utils.retries import RetryState def make_trace() -> vf.Trace: @@ -175,8 +173,6 @@ def make_rollout( run._relay = FakeRelay() # type: ignore[assignment] run._secret = "box-secret" run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) - run._retryable = lambda error: isinstance(error, vf.ProviderError) - run.rescued = False run._durable = {} run._captures = set() run._closed = False @@ -305,50 +301,18 @@ async def test_no_snapshot_at_turns_by_default(): assert backend.taken == 0 -async def test_a_failed_model_call_that_the_run_retries_takes_a_snapshot(): - run, backend, session = make_rollout() - run._checkpointing = RolloutCheckpointConfig() - # The run would not retry this error: the box is not stopped for it. - await run._rescue(vf.HarnessError("exit 1")) - assert backend.taken == 0 - await run._rescue(vf.ProviderError("overloaded", status_code=529)) - (checkpoint,) = run.trace.checkpoints - assert backend.taken == 1 and checkpoint.node == len(run.trace.nodes) - 1 - assert run.rescued - assert session.flowing.is_set() - # The same call fails again: the snapshot already holds the box that waits for it. - await run._rescue(vf.ProviderError("overloaded", status_code=529)) +async def test_a_call_asked_again_takes_no_second_snapshot(): + run, backend, _ = make_rollout(interval=1) + first = await run._checkpoint() + assert first is not None and backend.taken == 1 + # The same call comes again (the snapshot reset the box's connections). + assert await run._checkpoint() is None assert backend.taken == 1 - # A turn later, a new failure needs a new snapshot. - run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) - await run._rescue(vf.ProviderError("overloaded", status_code=529)) - assert backend.taken == 2 - - -async def test_a_run_without_a_retry_left_takes_no_snapshot(): - run, backend, _ = make_rollout() - run._retryable = None - await run._rescue(vf.ProviderError("overloaded", status_code=529)) - assert backend.taken == 0 - - -def test_retry_state_tells_whether_an_error_gets_a_retry_without_spending_it(): - state = RetryState( - RetryConfig(max_retries=0, rules=[{"type": "ProviderError", "max_retries": 1}]) - ) - error = vf.ProviderError("overloaded", status_code=529) - assert state.allows(error) and state.allows(error) - assert not state.allows(vf.HarnessError("exit 1")) - trace = make_trace() - trace.record_error(error) - assert state.next_error(trace.errors) is not None - assert not state.allows(error) def test_checkpointing_is_on_by_default(): assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() assert RolloutCheckpointConfig().interval is None - assert not RolloutCheckpointConfig().on_error assert gated("bash", vf.PrimeConfig())._checkpointing() is not None @@ -541,7 +505,8 @@ async def test_an_owners_snapshot_outlives_the_rollout(): await settling assert await saving is not None (pinned,) = run.trace.checkpoints - # A newer snapshot of the run's own does not prune the owner's. + # A newer snapshot of the run's own, a turn later, does not prune the owner's. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) newest = await run._checkpoint() await run._settle_checkpoints(wait=True) assert backend.deleted == [] diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 71e4343e8e..a5dc143226 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -39,7 +39,6 @@ class FakeRollout: def __init__(self, *, on_trace, trace_id, attempt, **_): self.rewound_snapshots: list[str] = [] self.deleted_snapshots: set[str] = set() - self.rescued = False self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) @@ -81,7 +80,6 @@ def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): restores.append(None if restore is None else restore[1].id) self.rewound_snapshots: list[vf.Snapshot] = [] self.deleted_snapshots: set[str] = set() - self.rescued = False self.restored_turns = None if restore is None: self.trace = make_trace(id=trace_id, attempt=attempt) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 0bc11c090d..7b4a60037c 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -796,8 +796,6 @@ async def run( attempt=first_attempt + attempt + 1, restore=restoring_from, checkpoint=checkpointing, - # The retry after this attempt is still to be granted. - retryable=(retry_state.allows if attempt < budget else None), ) trace = run.trace snapshots.update((s.id, s) for s in run.rewound_snapshots) @@ -812,16 +810,13 @@ async def run( any(error.stage == "setup" for error in trace.errors[seen:]) or ( not trace.ok - and not run.rescued and run.restored_turns is not None and trace.num_turns <= run.restored_turns ) ): # The restore led nowhere: the box never ran from it, or the # attempt failed before its first new turn. The next retry - # steps back to an older snapshot, or starts over. A model - # call that failed again is not the snapshot's fault: its - # retry restores the same box (`Rollout.rescued`). + # steps back to an older snapshot, or starts over. failed_restores.add(target.id) if trace.ok: break @@ -922,7 +917,6 @@ async def _run_once( attempt: int, restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, - retryable: Callable[[Exception], bool] | None = None, ) -> Rollout: if collect_artifacts: _check_artifact_runtime(params["runtime_config"]) @@ -934,7 +928,6 @@ async def _run_once( trace_id=trace_id, restore=restore, checkpoint=checkpoint, - retryable=retryable, attempt=attempt, **params, ) diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 34eb5858fe..3c34215b99 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -44,20 +44,14 @@ class RolloutCheckpointConfig(BaseConfig): another process continues the rollout from there instead of starting over. The restored box continues the harness program itself and asks again for the model call that was open. By default a rollout takes a snapshot only when its owner - saves it (`save_all`); `on_error` and `interval` add snapshots the rollout takes - by itself. A snapshot stops the box for a few seconds. It needs a runtime that - takes memory snapshots + saves it (`save_all`); `interval` adds snapshots the rollout takes by itself. A + snapshot stops the box for a few seconds and resets its open connections to the + outside. It needs a runtime that takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services of its own. Elsewhere a retry starts over. Every snapshot is deleted once the run is final.""" - on_error: bool = False - """Snapshot when a model call fails with an error the run retries, before the - harness gets the error: the retry continues the box that still waits for the - reply. It helps when the harness would end on the error; a harness that handles a - failed call itself (its SDK retries, or a sub-agent is started again) never uses - the snapshot.""" interval: PositiveInt | None = None """Snapshot at every n-th model turn (the n-th, the 2n-th, ...), for failures that leave no box to snapshot (the box died, the harness program crashed): a retry diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 3a4e07e170..10f01490fd 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1045,20 +1045,6 @@ async def sample() -> web.Response: else None ) - async def rescue(failure: Exception) -> None: - """Snapshot the box before the harness learns of `failure`: a retry - then continues a box that still waits for this turn's reply.""" - if session.rescuer is None: - return - if capture is not None: - await asyncio.wait({capture}) - try: - await session.rescuer(failure) - except Exception: - logger.warning( - "checkpoint failed: id=%s", session.trace.id, exc_info=True - ) - try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1135,7 +1121,6 @@ async def rescue(failure: Exception) -> None: logger.warning( "Model call failed - %s: %s", type(e).__name__, clip(str(e)) ) - await rescue(e) return _error_response( dialect, str(e), status=getattr(e, "status_code", 502) ) @@ -1144,7 +1129,6 @@ async def rescue(failure: Exception) -> None: logger.warning( "Model call failed - %s: %s", type(e).__name__, clip(str(e)) ) - await rescue(e) return _error_response(dialect, str(e), status=502) except BaseException as e: # A cancelled exchange (harness disconnect, shutdown) is still diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 92d7ce0ac3..dc9eae5c8a 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -167,7 +167,6 @@ def __init__( attempt: int = 1, restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, - retryable: Callable[[Exception], bool] | None = None, ) -> None: """`attempt` numbers this run among the trace's retries, from 1 (stamped on the trace and the errors it captures). `restore` continues a failed attempt's trace from one of @@ -197,11 +196,6 @@ def __init__( self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts self._checkpointing = checkpoint - self._retryable = retryable - """Whether the run's owner would retry it after this error (`Agent.run`).""" - self.rescued = False - """Whether the newest snapshot holds the box as it waited for a model call that - then failed (`_rescue`): a retry restores it and asks for the call again.""" self._durable: dict[str, asyncio.Task[None]] = {} self._captures: set[asyncio.Task] = set() """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" @@ -529,8 +523,6 @@ async def open(self) -> bool: and self._memory ): self._session.checkpointer = self._checkpoint - if self._checkpointing.on_error: - self._session.rescuer = self._rescue logger.debug( "Set up %s harness in %s", self.harness.config.name, @@ -936,28 +928,15 @@ async def _capture(self) -> Checkpoint | None: ) ): return None - return await self._take_snapshot() - - async def _rescue(self, error: Exception) -> None: - """Snapshot the box whose model call just failed with `error`, if a retry of - the run would follow. The box still waits for the call's reply, so the box a - retry restores asks for it again.""" - if self._closed or self._retryable is None or not self._retryable(error): - return newest = next( (c for c in reversed(self.trace.checkpoints) if not c.deleted), None ) if newest is not None and newest.node == self._last_node(): - # No turn since the newest snapshot: it holds this same open call. - self.rescued = True - return - task = asyncio.current_task() - if task is not None: - self._captures.add(task) - try: - self.rescued = await self._take_snapshot() is not None - finally: - self._captures.discard(task) + # No turn since the newest snapshot, so it holds this same open call. A + # snapshot resets the box's connections and its relay asks again: a call + # that keeps failing would otherwise take a snapshot per request. + return None + return await self._take_snapshot() def _last_node(self) -> int | None: return len(self.trace.nodes) - 1 if self.trace.nodes else None diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index a550b5e652..493464bac0 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -232,10 +232,6 @@ class RolloutSession: the reply waits for the capture (`Rollout` sets it when the rollout takes snapshots). Returns the checkpoint it took, if any.""" - rescuer: Callable[[Exception], Awaitable[None]] | None = None - """Snapshots the rollout's box when a model call failed, before the harness gets - the error (`Rollout` sets it when the rollout takes snapshots).""" - flowing: asyncio.Event = field(default_factory=asyncio.Event, repr=False) """Cleared while a memory snapshot of the box is taken (`hold`): nothing the box asked for is answered or written until it is set again, so the box the snapshot diff --git a/verifiers/v1/utils/retries.py b/verifiers/v1/utils/retries.py index e93bd82ec8..7870325d45 100644 --- a/verifiers/v1/utils/retries.py +++ b/verifiers/v1/utils/retries.py @@ -95,44 +95,26 @@ def next_error(self, errors: Iterable[Error]) -> Error | None: Return the triggering error so logs identify the actual retry cause. """ for error in errors: - index = self._rule(error) - if index is not None and self.used[index] < self.rules[index].max_retries: - self.used[index] += 1 - return error - return None - - def allows(self, error: Exception) -> bool: - """Whether `next_error` would grant a retry for `error` now. Consumes nothing.""" - from verifiers.v1.trace import Error - - index = self._rule( - Error( - type=type(error).__name__, - message=str(error), - status_code=getattr(error, "status_code", None), - ) - ) - return index is not None and self.used[index] < self.rules[index].max_retries - - def _rule(self, error: Error) -> int | None: - """The first rule that matches `error`: it alone sets the error's budget.""" - for index, rule in enumerate(self.rules): - if rule.type is not None and error.type != rule.type: - continue - if rule.status_code is not None and ( - error.status_code is None - or not any( - error.status_code == status - if isinstance(status, int) - else error.status_code // 100 == int(status[0]) - for status in rule.status_code - ) - ): - continue - pattern = self.patterns[index] - if pattern is not None and pattern.search(error.message) is None: - continue - return index + for index, rule in enumerate(self.rules): + if rule.type is not None and error.type != rule.type: + continue + if rule.status_code is not None and ( + error.status_code is None + or not any( + error.status_code == status + if isinstance(status, int) + else error.status_code // 100 == int(status[0]) + for status in rule.status_code + ) + ): + continue + pattern = self.patterns[index] + if pattern is not None and pattern.search(error.message) is None: + continue + if self.used[index] < rule.max_retries: + self.used[index] += 1 + return error + break return None From 93ff84b9a144e4a093be9128d4e4b1b53eaaed00 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 02:22:37 +0000 Subject: [PATCH 46/53] test that every saved turn continues to the same result The restore tests continued a rollout from one point and cut the trace back at most one turn. This one saves a rollout at three turns and continues each saved trace: with a scripted model, every continuation must end with the messages of the first run. A counter that a process holds only in memory shows a command that ran twice or was skipped. Co-Authored-By: Claude --- tests/integration/fixtures/tick_restore.py | 53 ++++++++ tests/integration/test_checkpoint.py | 136 +++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 tests/integration/fixtures/tick_restore.py diff --git a/tests/integration/fixtures/tick_restore.py b/tests/integration/fixtures/tick_restore.py new file mode 100644 index 0000000000..0034ee5e88 --- /dev/null +++ b/tests/integration/fixtures/tick_restore.py @@ -0,0 +1,53 @@ +"""tick-restore: a task whose every command takes the next number from a counter that a +process holds only in its memory, and appends it to a file. The file ends as 1..N only +if every command ran exactly once in a box that kept the process: a command that ran +twice, was skipped, or met a new process leaves another sequence. + +Each command prints the file, so the tool results of two runs are equal only if the +boxes went through the same states. The command first sleeps, so a save requested when +a turn lands on the trace stops the box while that turn's command runs. +""" + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +TURNS = 4 +FIFO = "tick.fifo" +LOG = "ticks.txt" +HOLD_SECONDS = 3 +SERVER = ( + f"i=0; rm -f {FIFO}; mkfifo {FIFO}; " + f'while :; do i=$((i+1)); printf "%s\\n" "$i" > {FIFO}; sleep 0.2; done' +) +COMMAND = f"sleep {HOLD_SECONDS}; timeout 20 cat {FIFO} >> {LOG}; cat {LOG}" +EXPECTED = "\n".join(str(i) for i in range(1, TURNS + 1)) + + +class TickRestoreTask(vf.Task): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + await runtime.run_background(["sh", "-c", SERVER], {}, "server.log") + + @vf.reward(weight=1.0) + async def counted(self, trace: vf.Trace, runtime: Runtime) -> float: + """The file holds 1..TURNS, in order.""" + try: + result = await runtime.run(["sh", "-c", f"cat {LOG}"], {}) + except (SandboxError, OSError): + return 0.0 + return float(result.stdout.strip() == EXPECTED) + + +class TickRestoreTaskset(vf.Taskset[TickRestoreTask, vf.TasksetConfig]): + def load(self) -> list[TickRestoreTask]: + return [ + TickRestoreTask( + vf.TaskData( + idx=0, + prompt=f"Run `{COMMAND}` {TURNS} times, one command per turn.", + ) + ) + ] + + +__all__ = ["TickRestoreTaskset"] diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index 100250e296..441d3f9e95 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -10,6 +10,7 @@ snapshot.""" import asyncio +import json import pickle import pytest @@ -17,8 +18,10 @@ from grid import grid from prime_sandboxes import AsyncSandboxClient from subagent_restore import sub_agent_turns +from tick_restore import COMMAND, TURNS from verifiers.v1.clients import ModelContext +from verifiers.v1.dialects.chat import ChatDialect from verifiers.v1.env import RunSlot from verifiers.v1.restore import restore_entry, save_all from verifiers.v1.runtimes import PrimeBackend, Snapshot @@ -46,6 +49,66 @@ async def recording(self): return snapshots +@pytest.fixture +async def command_model(): + """A scripted model for the bash harness: its k-th reply asks for the tick task's + command, and the reply after the last command says done. A box that asks again for + a turn gets the same reply. Yields a client config that points at it.""" + from aiohttp import web + + dialect = ChatDialect() + + async def answer(request: web.Request) -> web.StreamResponse: + body = await request.json() + turn = sum(m.get("role") == "assistant" for m in body["messages"]) + 1 + if turn <= TURNS: + call = { + "id": f"call_{turn}", + "type": "function", + "function": { + "name": "bash", + "arguments": json.dumps({"command": COMMAND}), + }, + } + message = {"role": "assistant", "content": None, "tool_calls": [call]} + else: + message = {"role": "assistant", "content": "done"} + raw = { + "id": f"chatcmpl-{turn}", + "object": "chat.completion", + "created": 0, + "model": str(body.get("model", "scripted")), + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if turn <= TURNS else "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + if not dialect.streaming(body): + return web.json_response(raw) + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + for event in dialect.stream_events(raw): + await response.write(event) + await response.write_eof() + return response + + app = web.Application() + app.router.add_post("/{tail:.*}", answer) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + try: + yield {"base_url": f"http://127.0.0.1:{port}/v1"} + finally: + await runner.cleanup() + + async def platform_state(client: AsyncSandboxClient, snapshot_id: str) -> str | None: """The snapshot's state on prime, or None once it is gone.""" try: @@ -281,3 +344,76 @@ async def test_another_owner_continues_running_sub_agents( (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_every_saved_turn_continues_to_the_same_result( + taken, harness, harness_runtime, command_model +): + """One rollout is saved at its first, second and third turn and then finishes. A + second owner continues each saved trace, and every continuation ends with the + messages of the first run: a restore from any turn loses no command and repeats + none. The model is scripted, so the turns after a restore are the same ones.""" + config = _eval_config( + "tick-restore", + harness=harness, + runtime=harness_runtime, + max_turns=12, + max_tokens=4096, + rollout_timeout=300, + client=command_model, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + env = load_environment(config.env) + (task,) = list(env.taskset) + points = [] + async with env.serving(): + (slot,) = env.slots(task) + running = asyncio.ensure_future(env.run_slot(slot, ctx)) + try: + for turn in range(1, TURNS): + async with asyncio.timeout(300): + while not (slot.traces and slot.traces[0].num_turns >= turn): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.1) + (point,) = await save_all() + points.append(pickle.loads(pickle.dumps(point))) + episode = await running + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + (reference,) = episode.traces + assert reference.ok, [error.message for error in reference.errors] + assert reference.reward == 1.0 + + def messages(trace): + return [node.message.model_dump(exclude_none=True) for node in trace.nodes] + + async def continued(point): + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot( + RunSlot(task, id=slot.id, restore=point), ctx + ) + (trace,) = episode.traces + return trace + + # The saves are at three different turns. + saved_turns = [sum(node["sampled"] for node in p["nodes"]) for p in points] + assert saved_turns == sorted(set(saved_turns)), saved_turns + for point, trace in zip(points, await asyncio.gather(*map(continued, points))): + assert trace.ok, [error.message for error in trace.errors] + assert trace.has_restored + assert trace.reward == 1.0 + assert sum(node.parent is None for node in trace.nodes) == 1 + assert messages(trace) == messages(reference) + snapshots = [Snapshot(id=restore_entry(point)["id"]) for point in points] + assert sorted(await PrimeBackend.discard(snapshots)) == sorted( + snapshot.id for snapshot in snapshots + ) + await assert_no_leaks(taken) From fa84fa5369524f40c200385fa69bd1fbdb9a3677 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:53:21 +0000 Subject: [PATCH 47/53] keep a box running when the wait for its pause fails A pause that the platform accepted stops the box. When the wait for the paused state then failed, the box was never resumed and its snapshot was dropped from the list that teardown frees. One failed batched status call fails every box that waits in it, so a save of many rollouts could stop them all for good. Resume the box whatever the wait came to, keep the snapshot for teardown, and ask again when a status poll fails. Co-Authored-By: Claude --- verifiers/v1/runtimes/backends/prime.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index c69e2512a8..3030abdc2c 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -398,13 +398,15 @@ async def create() -> Snapshot: try: paused = await self._client.pause(self.info.id) snapshot = Snapshot(id=paused.checkpoint_id) + # Kept whatever comes next: a pause that was not seen to finish can + # still have made the snapshot. self.stray_snapshots.append(snapshot) try: await self._settle("PAUSED", "PAUSE_FAILED") - except BaseException: - self.stray_snapshots.remove(snapshot) - raise - await self._resume() + finally: + # The pause was accepted, so the box stops or has stopped. It must + # run again also when the wait for it failed. + await self._resume() finally: self._awake.set() return snapshot @@ -464,7 +466,16 @@ async def _settle(self, status: str, failure: str) -> None: if shared.poller is None: shared.poller = _StatusPoller(shared.client) while True: - box = await shared.poller.next(self.info.id) + try: + box = await shared.poller.next(self.info.id) + except Exception as e: + # One failed batch fails every box that waits in it: ask again. + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status}: {e}" + ) from e + await asyncio.sleep(_STATUS_POLL_SECONDS) + continue if box.status == status: return if box.error_type == failure or box.status in _DEAD_STATUSES: From a1b88d85d5baf63694621d01e3a52af3d1238952 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:53:22 +0000 Subject: [PATCH 48/53] tell the server's own 404 from a tunnel edge's page The relay took every 404, 502, 503 or 504 that was not JSON or SSE for the error page of a tunnel edge and retried it for five minutes. The interception server answers an unknown path with a plain-text 404, so a harness that asked for one waited that long. The server now marks each of its responses with a header, and only a response without it counts as the edge's. Co-Authored-By: Claude --- tests/unit/interception/test_relay.py | 26 +++++++++++++++++++++++ verifiers/v1/interception/relay/server.py | 11 +++++----- verifiers/v1/interception/server.py | 9 ++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 5c6e3ae00a..9e27385428 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -21,6 +21,7 @@ import pytest from verifiers.v1.interception.relay import RELAY_SOURCE, relay_config +from verifiers.v1.interception.server import SERVER_HEADER TIMEOUT = 5 @@ -75,6 +76,7 @@ def close(self) -> None: def reply(handler, status: int, body: dict) -> None: data = json.dumps(body).encode() handler.send_response(status) + handler.send_header(SERVER_HEADER, "1") handler.send_header("Content-Type", "application/json") handler.send_header("Content-Length", str(len(data))) handler.end_headers() @@ -436,6 +438,30 @@ def test_interception_errors_reach_the_harness(upstreams, make_relay): assert a.requests.qsize() == 1 +def test_the_servers_own_404_reaches_the_harness(upstreams, make_relay): + """An unknown path is the server's answer, not a tunnel edge's page: the same + status and body type, told apart by the server's header.""" + a, _ = upstreams + + def respond(handler) -> None: + body = b"404: Not Found" + handler.send_response(404) + handler.send_header(SERVER_HEADER, "1") + handler.send_header("Content-Type", "text/plain") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/unknown", b"{}", {}) + response = connection.getresponse() + assert response.status == 404 + assert response.read() == b"404: Not Found" + assert a.requests.qsize() == 1 + + def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): """The host replaces a lost tunnel: the request waits on the dead URL and goes through once the relay is rerouted to the new one.""" diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index 6c7561d4d9..f90243e89e 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -53,8 +53,10 @@ the host to notice a lost tunnel, open a new one and reroute the relay.""" EDGE_STATUSES = frozenset({404, 502, 503, 504}) """Statuses a tunnel edge answers with its own page when it cannot reach the upstream -("404 Tunnel Not Found", a bad gateway). The interception server answers with JSON or -SSE, so only a response of another type with one of these statuses is the edge's.""" +("404 Tunnel Not Found", a bad gateway).""" +SERVER_HEADER = "x-vf-interception" +"""Header on every response of the interception server, its own 404 for an unknown path +included. A response with an edge status and without it never reached the server.""" BACKOFF_SECONDS = 0.5 BACKOFF_CAP_SECONDS = 10.0 CONNECT_TIMEOUT_SECONDS = 10.0 @@ -239,9 +241,8 @@ class UpstreamError(Exception): def is_edge_error(response: http.client.HTTPResponse) -> bool: """Whether `response` is the tunnel edge's own error page, not the upstream's answer.""" - content_type = (response.getheader("Content-Type") or "").lower() - return response.status in EDGE_STATUSES and not content_type.startswith( - ("application/json", "text/event-stream") + return ( + response.status in EDGE_STATUSES and response.getheader(SERVER_HEADER) is None ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 10f01490fd..d7b48e2df6 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -93,6 +93,10 @@ # Attempt counter the stainless-generated SDKs (OpenAI, Anthropic) send on every request: # 0 on the first attempt, incremented on each retry of the same request. RETRY_COUNT_HEADER = "x-stainless-retry-count" +SERVER_HEADER = "x-vf-interception" +"""Marks every response as this server's own, so a box's relay can tell an answer of +the server (a 404 for an unknown path) from the error page of a tunnel edge that could +not reach it (`relay/server.py`).""" IDEMPOTENCY_KEY_HEADER = "Idempotency-Key" IDEMPOTENCY_CACHE_TTL_SECONDS = 600 IDEMPOTENCY_CACHE_MAX_COMPLETED = 64 @@ -137,6 +141,10 @@ async def _request_digest(raw: bytes) -> bytes: return await asyncio.to_thread(_body_digest, raw) +async def _mark_response(request: web.Request, response: web.StreamResponse) -> None: + response.headers[SERVER_HEADER] = "1" + + def _error_response(dialect: Dialect, message: str, *, status: int) -> web.Response: """Keep the native error classification consistent with its HTTP status.""" return web.json_response( @@ -648,6 +656,7 @@ async def start(self) -> None: app = web.Application( client_max_size=MAX_REQUEST_BODY, middlewares=[_isolate_log_scope] ) + app.on_response_prepare.append(_mark_response) app.router.add_get(HEALTH_PATH, self.handle_health) for dialect in DIALECTS: for route in dialect.routes: From 2537fb03662eab0655aef3c3b5b2d073d9f90aed Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:53:22 +0000 Subject: [PATCH 49/53] keep a hold and a snapshot from undoing each other Three cases let a box see a reply that its checkpoint does not record, or left a rollout stopped: - A release (the owner's, or the timer) answered the box while another snapshot had it stopped. The snapshot now ends the hold itself when it is done. - A save that was cancelled or failed kept the hold with no timer and left its snapshot pinned for an owner that never got it. - A snapshot of a turn that waited for another snapshot recorded the trace with that turn's reply, which the box had not got. It is now skipped. A save without a hold no longer ends the hold of an earlier one. Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 40 +++++++++++++++++++++++++++++++++ verifiers/v1/rollout.py | 42 ++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index fb02187174..8d0c78930d 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -444,6 +444,46 @@ async def test_a_held_save_answers_the_box_only_after_release(): assert session.flowing.is_set() and run._release_timer is None +async def test_a_release_does_not_answer_a_box_that_another_snapshot_stops(): + run, backend, session = make_rollout() + backend.running.set() + assert await run.save(hold=True) is not None + # A turn later the rollout takes a snapshot of its own, and the owner releases + # while the box is stopped for it. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + backend.running.clear() + backend.stopped.clear() + capture = asyncio.create_task(run._checkpoint()) + await backend.stopped.wait() + run.release() + assert not session.flowing.is_set() + backend.running.set() + assert await capture is not None + assert session.flowing.is_set() + + +async def test_a_cancelled_held_save_does_not_hold_the_box(): + run, backend, session = make_rollout() + saving = asyncio.create_task(run.save(hold=True)) + await backend.stopped.wait() + saving.cancel() + assert (await asyncio.gather(saving, return_exceptions=True))[0] is not None + assert saving.cancelled() + assert session.flowing.is_set() + assert not run._held and run._release_timer is None + + +async def test_a_snapshot_that_waited_past_its_turn_is_not_taken(): + run, backend, session = make_rollout() + async with run._capturing: + capture = asyncio.create_task(run._checkpoint()) + await asyncio.sleep(0) + # The turn's reply lands on the trace while the snapshot waits its turn. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + assert await capture is None + assert backend.taken == 0 and session.flowing.is_set() + + async def test_a_failed_held_save_does_not_hold_the_box(): run, backend, session = make_rollout() diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index dc9eae5c8a..84cbf25da5 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from types import EllipsisType from typing import Any, Literal from urllib.parse import urlsplit @@ -936,7 +937,7 @@ async def _capture(self) -> Checkpoint | None: # snapshot resets the box's connections and its relay asks again: a call # that keeps failing would otherwise take a snapshot per request. return None - return await self._take_snapshot() + return await self._take_snapshot(at=self._last_node()) def _last_node(self) -> int | None: return len(self.trace.nodes) - 1 if self.trace.nodes else None @@ -965,21 +966,33 @@ async def save(self, *, hold: bool = False) -> dict[str, Any] | None: task = asyncio.current_task() if task is not None: self._captures.add(task) - self._held = hold + # A save without a hold does not end the hold of an earlier one. + self._held = hold or self._held + checkpoint = None try: checkpoint = await self._take_snapshot(pinned=True) + if checkpoint is None: + self.release() + self.save_skipped = "its memory snapshot failed" + return None + saved = restorable(self.trace, checkpoint) + except BaseException: + # The owner never gets this save: nobody would release the hold or delete + # the snapshot, so both go back to the rollout. + if checkpoint is not None: + checkpoint.pinned = False + self.release() + raise finally: self._captures.discard(task) - if checkpoint is None: - self.release() - self.save_skipped = "its memory snapshot failed" - return None if hold: + if self._release_timer is not None: + self._release_timer.cancel() # An owner that never releases must not stall the rollout for good. self._release_timer = asyncio.get_running_loop().call_later( HOLD_SECONDS, self.release ) - return restorable(self.trace, checkpoint) + return saved def release(self) -> None: """Answer the box again after a `save(hold=True)`.""" @@ -987,19 +1000,26 @@ def release(self) -> None: if self._release_timer is not None: self._release_timer.cancel() self._release_timer = None - self._session.resume() + # A snapshot that is being taken holds the box for itself, and lets the + # replies flow when it is done. + if not self._capturing.locked(): + self._session.resume() - async def _take_snapshot(self, *, pinned: bool = False) -> Checkpoint | None: + async def _take_snapshot( + self, *, pinned: bool = False, at: int | None | EllipsisType = ... + ) -> Checkpoint | None: """Snapshot the box with its memory. Nothing the box asked for is answered while the snapshot is taken (`RolloutSession.hold`), so the box it stops has seen exactly the trace as recorded here; a box restored from it asks again - for what was still open.""" + for what was still open. With `at`, only while the trace still ends at that + node: a snapshot of a turn that waited for another one comes too late when + the turn's reply is on the trace and not yet in the box.""" assert self.runtime is not None and self._relay is not None session = self._session if self._harness_session is None: return None async with self._capturing: - if self._closed: + if self._closed or (at is not ... and at != self._last_node()): return None session.hold() try: From 78a84360a7682de67e0e27e3d76f755b7eb6cb90 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:53:22 +0000 Subject: [PATCH 50/53] pin prime-sandboxes to the attach branch The harness program now runs in a session and a restored box is attached to by session id, which needs `attach_process`. Remove the pin when a release has it. Co-Authored-By: Claude --- pyproject.toml | 2 ++ uv.lock | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b1e46ee16e..046a899282 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,8 @@ url = "https://pypi.org/simple" default = true [tool.uv.sources] +# Temporary, until a release has `attach_process` (PrimeIntellect-ai/prime#1147). +prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime", rev = "fae6d0bc91717b95ff1b109d93d58a22e3408df2", subdirectory = "packages/prime-sandboxes" } compact = { path = "environments/compact", editable = true } glossary = { path = "environments/glossary", editable = true } deepwiki = { path = "environments/deepwiki", editable = true } diff --git a/uv.lock b/uv.lock index e2a1b7e7cb..80697900e2 100644 --- a/uv.lock +++ b/uv.lock @@ -3263,8 +3263,8 @@ toml = [ [[package]] name = "prime-sandboxes" -version = "0.5.9" -source = { registry = "https://pypi.org/simple" } +version = "0.5.11" +source = { git = "https://github.com/PrimeIntellect-ai/prime?subdirectory=packages%2Fprime-sandboxes&rev=fae6d0bc91717b95ff1b109d93d58a22e3408df2#fae6d0bc91717b95ff1b109d93d58a22e3408df2" } dependencies = [ { name = "aiofiles" }, { name = "certifi" }, @@ -3275,10 +3275,23 @@ dependencies = [ { name = "pyqwest" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8d/50/40e351b1fada75c8a166c3e482584912e660d7a143b5a1fa317c2038f2d0/prime_sandboxes-0.5.9.tar.gz", hash = "sha256:4aab8f2ca2dbd1f18cf356bb8c072baa29747cdee0305817d4d4eed7283c8cae", size = 173127, upload-time = "2026-11-10T01:03:33.691Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/77/888edf3076ef91d424f323aad4455e4a2c426cf1dde91728a46e19a029ac/prime_sandboxes-0.5.9-py3-none-any.whl", hash = "sha256:f66355ab2c8384d1ade329c82b9497165c490778031f927c00fefa6101f1536d", size = 93862, upload-time = "2026-11-10T01:03:31.814Z" }, -] + +[package.metadata] +requires-dist = [ + { name = "aiofiles", specifier = ">=23" }, + { name = "certifi" }, + { name = "connectrpc", specifier = ">=0.11.1,<0.12" }, + { name = "httpx", specifier = ">=0.25" }, + { name = "protobuf", specifier = ">=6.31.1" }, + { name = "pydantic", specifier = ">=2" }, + { name = "pyqwest", specifier = ">=0.6" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.13.1" }, + { name = "tenacity", specifier = ">=8" }, +] +provides-extras = ["dev"] [[package]] name = "prime-tunnel" From 0486bcecd703d1377ffe72f3cf901183e4449b28 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:54:28 +0000 Subject: [PATCH 51/53] stop the box in the cancelled-save test The fake backend finished its snapshot at once, so the save was never cancelled. Co-Authored-By: Claude --- tests/unit/test_checkpoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 8d0c78930d..d3f0af20e3 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -464,6 +464,7 @@ async def test_a_release_does_not_answer_a_box_that_another_snapshot_stops(): async def test_a_cancelled_held_save_does_not_hold_the_box(): run, backend, session = make_rollout() + backend.running.clear() saving = asyncio.create_task(run.save(hold=True)) await backend.stopped.wait() saving.cancel() From 167b4b614f213434caa56a62913dd9f9716cd38e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 19:53:22 +0000 Subject: [PATCH 52/53] tell the server's own 404 from a tunnel edge's page The relay took every 404, 502, 503 or 504 that was not JSON or SSE for the error page of a tunnel edge and retried it for five minutes. The interception server answers an unknown path with a plain-text 404, so a harness that asked for one waited that long. The server now marks each of its responses with a header, and only a response without it counts as the edge's. Co-Authored-By: Claude (cherry picked from commit a1b88d85d5baf63694621d01e3a52af3d1238952) --- tests/unit/interception/test_relay.py | 26 +++++++++++++++++++++++ verifiers/v1/interception/relay/server.py | 11 +++++----- verifiers/v1/interception/server.py | 9 ++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py index 5c6e3ae00a..9e27385428 100644 --- a/tests/unit/interception/test_relay.py +++ b/tests/unit/interception/test_relay.py @@ -21,6 +21,7 @@ import pytest from verifiers.v1.interception.relay import RELAY_SOURCE, relay_config +from verifiers.v1.interception.server import SERVER_HEADER TIMEOUT = 5 @@ -75,6 +76,7 @@ def close(self) -> None: def reply(handler, status: int, body: dict) -> None: data = json.dumps(body).encode() handler.send_response(status) + handler.send_header(SERVER_HEADER, "1") handler.send_header("Content-Type", "application/json") handler.send_header("Content-Length", str(len(data))) handler.end_headers() @@ -436,6 +438,30 @@ def test_interception_errors_reach_the_harness(upstreams, make_relay): assert a.requests.qsize() == 1 +def test_the_servers_own_404_reaches_the_harness(upstreams, make_relay): + """An unknown path is the server's answer, not a tunnel edge's page: the same + status and body type, told apart by the server's header.""" + a, _ = upstreams + + def respond(handler) -> None: + body = b"404: Not Found" + handler.send_response(404) + handler.send_header(SERVER_HEADER, "1") + handler.send_header("Content-Type", "text/plain") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/unknown", b"{}", {}) + response = connection.getresponse() + assert response.status == 404 + assert response.read() == b"404: Not Found" + assert a.requests.qsize() == 1 + + def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): """The host replaces a lost tunnel: the request waits on the dead URL and goes through once the relay is rerouted to the new one.""" diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py index 6c7561d4d9..f90243e89e 100644 --- a/verifiers/v1/interception/relay/server.py +++ b/verifiers/v1/interception/relay/server.py @@ -53,8 +53,10 @@ the host to notice a lost tunnel, open a new one and reroute the relay.""" EDGE_STATUSES = frozenset({404, 502, 503, 504}) """Statuses a tunnel edge answers with its own page when it cannot reach the upstream -("404 Tunnel Not Found", a bad gateway). The interception server answers with JSON or -SSE, so only a response of another type with one of these statuses is the edge's.""" +("404 Tunnel Not Found", a bad gateway).""" +SERVER_HEADER = "x-vf-interception" +"""Header on every response of the interception server, its own 404 for an unknown path +included. A response with an edge status and without it never reached the server.""" BACKOFF_SECONDS = 0.5 BACKOFF_CAP_SECONDS = 10.0 CONNECT_TIMEOUT_SECONDS = 10.0 @@ -239,9 +241,8 @@ class UpstreamError(Exception): def is_edge_error(response: http.client.HTTPResponse) -> bool: """Whether `response` is the tunnel edge's own error page, not the upstream's answer.""" - content_type = (response.getheader("Content-Type") or "").lower() - return response.status in EDGE_STATUSES and not content_type.startswith( - ("application/json", "text/event-stream") + return ( + response.status in EDGE_STATUSES and response.getheader(SERVER_HEADER) is None ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 7e74164740..30e993c303 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -92,6 +92,10 @@ # Attempt counter the stainless-generated SDKs (OpenAI, Anthropic) send on every request: # 0 on the first attempt, incremented on each retry of the same request. RETRY_COUNT_HEADER = "x-stainless-retry-count" +SERVER_HEADER = "x-vf-interception" +"""Marks every response as this server's own, so a box's relay can tell an answer of +the server (a 404 for an unknown path) from the error page of a tunnel edge that could +not reach it (`relay/server.py`).""" IDEMPOTENCY_KEY_HEADER = "Idempotency-Key" IDEMPOTENCY_CACHE_TTL_SECONDS = 600 IDEMPOTENCY_CACHE_MAX_COMPLETED = 64 @@ -136,6 +140,10 @@ async def _request_digest(raw: bytes) -> bytes: return await asyncio.to_thread(_body_digest, raw) +async def _mark_response(request: web.Request, response: web.StreamResponse) -> None: + response.headers[SERVER_HEADER] = "1" + + def _error_response(dialect: Dialect, message: str, *, status: int) -> web.Response: """Keep the native error classification consistent with its HTTP status.""" return web.json_response( @@ -647,6 +655,7 @@ async def start(self) -> None: app = web.Application( client_max_size=MAX_REQUEST_BODY, middlewares=[_isolate_log_scope] ) + app.on_response_prepare.append(_mark_response) app.router.add_get(HEALTH_PATH, self.handle_health) for dialect in DIALECTS: for route in dialect.routes: From 3be0b89dc7e9303458106698f0c7706f76b3d8d9 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 13 Nov 2026 20:36:45 +0000 Subject: [PATCH 53/53] probe which harnesses echo their replies Experiment only, not for merge: report per trace how many sampled replies the next request matched, and run only the multi-turn jobs. Co-Authored-By: Claude --- .github/workflows/tests.yml | 38 ------------------------------ tests/integration/conftest.py | 44 ++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 19409ffe18..7737ddd13e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -81,56 +81,18 @@ jobs: max-parallel: 16 matrix: include: - - behavior: Single-Turn - tests: tests/integration/test_single_turn.py - behavior: Multi-Turn tests: tests/integration/test_multi_turn.py - behavior: Multi-Turn Tool tests: tests/integration/test_multi_turn_tool.py - behavior: Stateful Tool tests: tests/integration/test_stateful_tool.py - - behavior: Shared Tool - tests: tests/integration/test_shared_tool.py - - behavior: Multimodal Tool - tests: tests/integration/test_multimodal_tool.py - - behavior: User Echo - tests: tests/integration/test_user_echo.py - behavior: Resume Tool tests: tests/integration/test_resume_tool.py - - behavior: Checkpoint Restore - tests: tests/integration/test_checkpoint.py - - behavior: User Sim - tests: tests/integration/test_user_sim.py - - behavior: Agent User - tests: tests/integration/test_agent_user.py - - behavior: Network Policy - tests: tests/integration/test_network_policy.py - - behavior: Service VMs - tests: tests/integration/test_service_scoring.py - - behavior: Native Compose - tests: tests/integration/test_native_compose.py - - behavior: Tool VMs - tests: tests/integration/test_tool_vms.py - - behavior: Mounts - tests: tests/integration/test_mounts.py - - behavior: Sampling - tests: tests/integration/test_sampling.py - - behavior: Parallel - tests: tests/integration/test_parallel.py - - behavior: Scoring Network - tests: tests/integration/test_scoring_network.py - - behavior: Skills - tests: tests/integration/test_skills.py - behavior: Limits tests: tests/integration/test_limits.py - - behavior: Rubric Judge - tests: tests/integration/test_rubric_judge.py - - behavior: Agentic Judge - tests: tests/integration/test_agentic_judge.py - behavior: Multi-Agent tests: tests/integration/test_multi_agent.py - - behavior: Examples - tests: tests/integration/test_examples.py env: PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b5be480d1b..90e5b3a91c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -177,6 +177,46 @@ def _eval_config( ) +def _echo_probe(taskset: str, episodes) -> None: + """Experiment: does the harness show each reply again in its next request, in a form + the graph matches? A sampled node with a child was matched. A sampled node without + one is a final reply, or a reply whose echo did not match: then its parent has + another child that is an assistant message the model did not sample.""" + import warnings + + for episode in episodes: + for trace in episode.traces: + nodes = trace.nodes + children: dict[int | None, list[int]] = {} + for index, node in enumerate(nodes): + children.setdefault(node.parent, []).append(index) + sampled = [i for i, node in enumerate(nodes) if node.sampled] + matched = [i for i in sampled if children.get(i)] + leaves = [i for i in sampled if not children.get(i)] + lost = [ + i + for i in leaves + if any( + j != i + and not nodes[j].sampled + and nodes[j].message.role == "assistant" + for j in children.get(nodes[i].parent, []) + ) + ] + unsampled_assistant = sum( + not node.sampled and node.message.role == "assistant" for node in nodes + ) + harness = trace.agent.config.harness + warnings.warn( + f"ECHO harness={harness.id if harness else None} taskset={taskset} " + f"dialect={getattr(harness, 'dialect', None)} sampled={len(sampled)} " + f"matched={len(matched)} leaves={len(leaves)} lost={len(lost)} " + f"unsampled_assistant={unsampled_assistant} " + f"branches={trace.num_branches} nodes={len(nodes)} ok={trace.ok}", + stacklevel=1, + ) + + @pytest.fixture def run(tmp_path): """Run a taskset's episodes in-process with `vf-run`'s runner, saving them under @@ -185,7 +225,9 @@ def run(tmp_path): async def _episodes(taskset: str, **kwargs) -> list[Episode]: config = _eval_config(taskset, **kwargs) config.output_dir = tmp_path - return await run_episodes(config) + episodes = await run_episodes(config) + _echo_probe(taskset, episodes) + return episodes return _episodes