diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 067a4883e7..7737ddd13e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -81,54 +81,18 @@ jobs: max-parallel: 16 matrix: include: - - behavior: Single-Turn - tests: tests/integration/test_single_turn.py - behavior: Multi-Turn tests: tests/integration/test_multi_turn.py - behavior: Multi-Turn Tool tests: tests/integration/test_multi_turn_tool.py - behavior: Stateful Tool tests: tests/integration/test_stateful_tool.py - - behavior: Shared Tool - tests: tests/integration/test_shared_tool.py - - behavior: Multimodal Tool - tests: tests/integration/test_multimodal_tool.py - - behavior: User Echo - tests: tests/integration/test_user_echo.py - behavior: Resume Tool tests: tests/integration/test_resume_tool.py - - behavior: User Sim - tests: tests/integration/test_user_sim.py - - behavior: Agent User - tests: tests/integration/test_agent_user.py - - behavior: Network Policy - tests: tests/integration/test_network_policy.py - - behavior: Service VMs - tests: tests/integration/test_service_scoring.py - - behavior: Native Compose - tests: tests/integration/test_native_compose.py - - behavior: Tool VMs - tests: tests/integration/test_tool_vms.py - - behavior: Mounts - tests: tests/integration/test_mounts.py - - behavior: Sampling - tests: tests/integration/test_sampling.py - - behavior: Parallel - tests: tests/integration/test_parallel.py - - behavior: Scoring Network - tests: tests/integration/test_scoring_network.py - - behavior: Skills - tests: tests/integration/test_skills.py - behavior: Limits tests: tests/integration/test_limits.py - - behavior: Rubric Judge - tests: tests/integration/test_rubric_judge.py - - behavior: Agentic Judge - tests: tests/integration/test_agentic_judge.py - behavior: Multi-Agent tests: tests/integration/test_multi_agent.py - - behavior: Examples - tests: tests/integration/test_examples.py env: PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} diff --git a/docs/agent.md b/docs/agent.md index db057cc754..ad3d6b73ee 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -72,3 +72,32 @@ max_turn_tokens = 4096 skip_on_truncated = true skip_on_timeout = true ``` + +## Retries and checkpoints + +`[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. + +Rollout checkpointing is on by default: a rollout can be snapshotted, and a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. + +By default a rollout takes a snapshot only when its owner saves it (`save_all`, below). `interval` adds snapshots that the rollout takes by itself, at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed. + +Without a snapshot, a retry starts over. + +A snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds. + +A rollout takes snapshots only when all of these hold. Otherwise a retry starts over, with a warning once per harness and runtime: + +- The runtime takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`: prime). +- The harness runs detached from the host (`SUPPORTS_MEMORY_RESTORE`: null, bash and the ACP harnesses). +- The rollout runs in a box of its own, not in a borrowed one. +- The rollout has no tool servers, no shared tools and no services of its own. They reach the host at an address that does not follow the box to its restore. + +A retry also starts over when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. + +```toml +[env.agent.checkpoint] +# interval = 10 # snapshot at the 10th, 20th, ... model turn (off by default) +keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest +``` + +An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns the trace of each as plain data the owner stores. The trace holds everything a restore needs, on the entry of its checkpoint. A later process continues a rollout from its saved trace with `RunSlot(task, id=..., restore=trace)` (or `Agent.run(task, restore=trace)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. A saved trace continues one rollout: an episode with more than one agent run cannot continue from the trace of one of them, and starts over. diff --git a/docs/architecture.md b/docs/architecture.md index 8e9057e599..9b2d55ca0c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -25,6 +25,17 @@ Routing traffic through the interception server enables several things beyond fo - Applying sampling parameters in harnesses that don't expose those settings directly - Running the task's [stop and interception hooks](tasksets.md#stops-and-interception) to inspect or rewrite tool responses and server-side web search results (for example, to block reward hacks) +### Relay + +A Prime tunnel can drop for a while (its client reconnects) or for good (the tunnel service drops its registration). A harness in a remote box therefore reaches the interception server through a relay in its own box, at `127.0.0.1`, not at the tunnel URL: + +- While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects. +- When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn. + +A box restored from a memory snapshot (`agent.checkpoint`) still runs its relay. The new rollout points that relay at its own interception server and maps the secret the harness holds to the new session's secret, so the harness keeps its URL and its secret. Model calls that were open when the snapshot stopped the box go to the new server. + +A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a uv script, like the harness programs and tool servers verifiers already runs in the box. A relay that does not start, gives up on a request or exits fails the rollout with a `RelayError`. + ## Finishing a rollout When the agent finishes its turn normally, the rollout collects artifacts and runs in-place grading before closing the harness session: diff --git a/docs/harnesses.md b/docs/harnesses.md index 54b1f418a1..383e69c6ee 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,6 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). +- `SUPPORTS_MEMORY_RESTORE`: indicates that a box restored from a memory snapshot continues the harness (`agent.checkpoint`). The program must run as a job of the box that the host only waits for (`runtime.run(..., program=current_program.get())`), keep no connection to the host open, and reach the host only through the endpoint it was given: the box's relay follows a restored box to its new interception server. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/pyproject.toml b/pyproject.toml index b1e46ee16e..046a899282 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,8 @@ url = "https://pypi.org/simple" default = true [tool.uv.sources] +# Temporary, until a release has `attach_process` (PrimeIntellect-ai/prime#1147). +prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime", rev = "fae6d0bc91717b95ff1b109d93d58a22e3408df2", subdirectory = "packages/prime-sandboxes" } compact = { path = "environments/compact", editable = true } glossary = { path = "environments/glossary", editable = true } deepwiki = { path = "environments/deepwiki", editable = true } diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b5be480d1b..90e5b3a91c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -177,6 +177,46 @@ def _eval_config( ) +def _echo_probe(taskset: str, episodes) -> None: + """Experiment: does the harness show each reply again in its next request, in a form + the graph matches? A sampled node with a child was matched. A sampled node without + one is a final reply, or a reply whose echo did not match: then its parent has + another child that is an assistant message the model did not sample.""" + import warnings + + for episode in episodes: + for trace in episode.traces: + nodes = trace.nodes + children: dict[int | None, list[int]] = {} + for index, node in enumerate(nodes): + children.setdefault(node.parent, []).append(index) + sampled = [i for i, node in enumerate(nodes) if node.sampled] + matched = [i for i in sampled if children.get(i)] + leaves = [i for i in sampled if not children.get(i)] + lost = [ + i + for i in leaves + if any( + j != i + and not nodes[j].sampled + and nodes[j].message.role == "assistant" + for j in children.get(nodes[i].parent, []) + ) + ] + unsampled_assistant = sum( + not node.sampled and node.message.role == "assistant" for node in nodes + ) + harness = trace.agent.config.harness + warnings.warn( + f"ECHO harness={harness.id if harness else None} taskset={taskset} " + f"dialect={getattr(harness, 'dialect', None)} sampled={len(sampled)} " + f"matched={len(matched)} leaves={len(leaves)} lost={len(lost)} " + f"unsampled_assistant={unsampled_assistant} " + f"branches={trace.num_branches} nodes={len(nodes)} ok={trace.ok}", + stacklevel=1, + ) + + @pytest.fixture def run(tmp_path): """Run a taskset's episodes in-process with `vf-run`'s runner, saving them under @@ -185,7 +225,9 @@ def run(tmp_path): async def _episodes(taskset: str, **kwargs) -> list[Episode]: config = _eval_config(taskset, **kwargs) config.output_dir = tmp_path - return await run_episodes(config) + episodes = await run_episodes(config) + _echo_probe(taskset, episodes) + return episodes return _episodes diff --git a/tests/integration/fixtures/memory_restore_v1.py b/tests/integration/fixtures/memory_restore_v1.py new file mode 100644 index 0000000000..76ce65f65b --- /dev/null +++ b/tests/integration/fixtures/memory_restore_v1.py @@ -0,0 +1,98 @@ +"""memory-restore: a task whose first attempt dies mid-rollout, so the retry restores the +box's newest memory snapshot and continues the harness program the box still runs. + +`setup` starts a process in the box that holds a nonce only in its memory and hands it +to whoever reads `nonce.fifo`. The nonce is on no disk, so only a box that continues +the process from a memory snapshot can answer. +The reward reads the nonce from the process. The task asks a harness with a shell for +two commands that build on each other, and a metric checks the file they leave. + +A response `@intercept` raises a `ProviderError` at the model turn `fail_turn`, for +every call of the first attempt: the harness gets the error as its reply and dies, +after the snapshot of that turn exists. The restored box still waits for that turn's +reply, gets it from the retry, and finishes: no new harness process, no notice to the +model. One failed call is not enough: the snapshot resets the box's connections, its +relay asks again, and a call that then succeeds would let the harness continue. +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +FIFO = "nonce.fifo" +STEP_ONE = "step1.txt" +DONE = "done.txt" +READ = f"timeout 10 cat {FIFO}" +# The nonce is the script's first argument: it lives in the process, not in a file. The +# sleep lets a reader see the end of one copy before the next write opens the FIFO. +SERVER = ( + f'rm -f {FIFO}; mkfifo {FIFO}; while :; do printf %s "$1" > {FIFO}; sleep 0.5; done' +) + + +class MemoryRestoreTaskConfig(vf.TaskConfig): + fail_turn: int = 2 + """The model turn whose reply fails once (from 1).""" + + +class MemoryRestoreConfig(vf.TasksetConfig): + task: MemoryRestoreTaskConfig = MemoryRestoreTaskConfig() + + +class MemoryRestoreTask(vf.Task[vf.TaskData, vf.State, MemoryRestoreTaskConfig]): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.run_background( + ["sh", "-c", SERVER, "sh", nonce], {}, "server.log" + ) + + @vf.intercept + def fail_once(self, response: vf.Response, trace: vf.Trace) -> None: + if trace.attempt == 1 and trace.num_turns + 1 >= self.config.fail_turn: + # A 4xx: harness SDKs end the turn at once instead of retrying it until the + # rollout's deadline. + raise vf.ProviderError("injected mid-rollout failure", status_code=400) + + async def _read(self, runtime: Runtime, command: str) -> str: + try: + result = await runtime.run(["sh", "-c", command], {}) + except (SandboxError, OSError): + return "" + return result.stdout.strip() + + @vf.reward(weight=1.0) + async def process_alive(self, trace: vf.Trace, runtime: Runtime) -> float: + """The process `setup` started still runs and still holds the nonce.""" + return float(await self._read(runtime, READ) == trace.info["nonce"]) + + @vf.metric + async def copied(self, trace: vf.Trace, runtime: Runtime) -> float: + """The agent's two commands left the nonce in the done file.""" + return float(await self._read(runtime, f"cat {DONE}") == trace.info["nonce"]) + + +class MemoryRestoreTaskset(vf.Taskset[MemoryRestoreTask, MemoryRestoreConfig]): + def load(self) -> list[MemoryRestoreTask]: + return [ + MemoryRestoreTask( + vf.TaskData( + idx=0, + prompt=( + "Run these shell commands in the working directory, one " + "command per turn, and wait for each result before the " + "next. If you cannot run commands, reply with the single " + "word: done.\n" + f"1. `{READ} > {STEP_ONE}`\n" + f"2. `cp {STEP_ONE} {DONE}`\n" + "3. Reply with the single word: done." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["MemoryRestoreTaskset"] diff --git a/tests/integration/fixtures/subagent_restore.py b/tests/integration/fixtures/subagent_restore.py new file mode 100644 index 0000000000..cf917f89e6 --- /dev/null +++ b/tests/integration/fixtures/subagent_restore.py @@ -0,0 +1,108 @@ +"""subagent-restore: a task whose agent hands two jobs to two sub-agents that run at the +same time, so a box snapshotted in the middle holds two running sub-agents. + +`setup` starts one process per sub-agent that holds a nonce only in its memory and +hands it to whoever reads its FIFO. Each sub-agent reads the nonce twice, one command +per turn. The second command first waits for the other sub-agent's first file and then +sleeps, so from the moment both sub-agents have a turn on the trace, neither finishes +for `HOLD_SECONDS`: a snapshot taken then catches both at work. The second read comes +from the process, so it works only in a box that kept its memory. +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +NAMES = ("alpha", "bravo") +ROOT_MARK = "COORDINATOR-TASK" +HOLD_SECONDS = 20 +SERVER = 'rm -f "$2"; mkfifo "$2"; while :; do printf %s "$1" > "$2"; sleep 0.5; done' + + +def mark(name: str) -> str: + """The text only `name`'s brief holds.""" + return f"SUBAGENT-{name.upper()}" + + +def brief(name: str) -> str: + other = NAMES[1 - NAMES.index(name)] + return ( + f"{mark(name)}. Run these shell commands in the working directory, one " + "command per turn, and wait for each result before the next.\n" + f"1. `timeout 10 cat {name}.fifo > {name}.1`\n" + f"2. `timeout 120 sh -c 'until [ -f {other}.1 ]; do sleep 1; done'; " + f"sleep {HOLD_SECONDS}; timeout 10 cat {name}.fifo > {name}.done`\n" + "3. Reply with the single word: done." + ) + + +def sub_agent_turns(trace: vf.Trace) -> dict[str, list[int]]: + """The sampled nodes of each sub-agent: those whose history holds the sub-agent's + brief and not the coordinator's task.""" + turns: dict[str, list[int]] = {name: [] for name in NAMES} + for index, node in enumerate(trace.nodes): + if not node.sampled: + continue + history, cursor = "", node.parent + while cursor is not None: + history += str(trace.nodes[cursor].message.model_dump()) + cursor = trace.nodes[cursor].parent + if ROOT_MARK in history: + continue + for name in NAMES: + if mark(name) in history: + turns[name].append(index) + return turns + + +class SubagentRestoreTask(vf.Task): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + for name in NAMES: + nonce = uuid.uuid4().hex + trace.info[name] = nonce + await runtime.run_background( + ["sh", "-c", SERVER, "sh", nonce, f"{name}.fifo"], {}, f"{name}.log" + ) + + async def _read(self, runtime: Runtime, command: str) -> str: + try: + result = await runtime.run(["sh", "-c", command], {}) + except (SandboxError, OSError): + return "" + return result.stdout.strip() + + @vf.reward(weight=1.0) + async def both_done(self, trace: vf.Trace, runtime: Runtime) -> float: + """Each sub-agent's second read left its nonce in its done file.""" + done = [ + await self._read(runtime, f"cat {name}.done") == trace.info[name] + for name in NAMES + ] + return float(all(done)) + + +class SubagentRestoreTaskset(vf.Taskset[SubagentRestoreTask, vf.TasksetConfig]): + def load(self) -> list[SubagentRestoreTask]: + briefs = "\n\n".join( + f"Brief for sub-agent {i + 1}:\n{brief(name)}" + for i, name in enumerate(NAMES) + ) + return [ + SubagentRestoreTask( + vf.TaskData( + idx=0, + prompt=( + f"{ROOT_MARK}. Do not run the commands below yourself. Start " + "exactly two sub-agents so that both run at the same time, " + "and give each one its brief word for word. Wait for both, " + "then reply with the single word: done.\n\n" + f"{briefs}" + ), + ) + ) + ] + + +__all__ = ["SubagentRestoreTaskset"] diff --git a/tests/integration/fixtures/tick_restore.py b/tests/integration/fixtures/tick_restore.py new file mode 100644 index 0000000000..0034ee5e88 --- /dev/null +++ b/tests/integration/fixtures/tick_restore.py @@ -0,0 +1,53 @@ +"""tick-restore: a task whose every command takes the next number from a counter that a +process holds only in its memory, and appends it to a file. The file ends as 1..N only +if every command ran exactly once in a box that kept the process: a command that ran +twice, was skipped, or met a new process leaves another sequence. + +Each command prints the file, so the tool results of two runs are equal only if the +boxes went through the same states. The command first sleeps, so a save requested when +a turn lands on the trace stops the box while that turn's command runs. +""" + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +TURNS = 4 +FIFO = "tick.fifo" +LOG = "ticks.txt" +HOLD_SECONDS = 3 +SERVER = ( + f"i=0; rm -f {FIFO}; mkfifo {FIFO}; " + f'while :; do i=$((i+1)); printf "%s\\n" "$i" > {FIFO}; sleep 0.2; done' +) +COMMAND = f"sleep {HOLD_SECONDS}; timeout 20 cat {FIFO} >> {LOG}; cat {LOG}" +EXPECTED = "\n".join(str(i) for i in range(1, TURNS + 1)) + + +class TickRestoreTask(vf.Task): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + await runtime.run_background(["sh", "-c", SERVER], {}, "server.log") + + @vf.reward(weight=1.0) + async def counted(self, trace: vf.Trace, runtime: Runtime) -> float: + """The file holds 1..TURNS, in order.""" + try: + result = await runtime.run(["sh", "-c", f"cat {LOG}"], {}) + except (SandboxError, OSError): + return 0.0 + return float(result.stdout.strip() == EXPECTED) + + +class TickRestoreTaskset(vf.Taskset[TickRestoreTask, vf.TasksetConfig]): + def load(self) -> list[TickRestoreTask]: + return [ + TickRestoreTask( + vf.TaskData( + idx=0, + prompt=f"Run `{COMMAND}` {TURNS} times, one command per turn.", + ) + ) + ] + + +__all__ = ["TickRestoreTaskset"] diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 66d6e5fe66..a9475fcab3 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,7 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). +`SUPPORTS_RESUME` or a native session). A `memory` need asks the runtime for +`SUPPORTS_MEMORY_SNAPSHOTS` and the harness for `Harness.SUPPORTS_MEMORY_RESTORE`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -66,8 +67,10 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", + "memory": "memory snapshots", } + # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -95,7 +98,10 @@ def assert_one_branch( f"no FORKS reason for {set(forks) - FORKS.keys()}" ) if harness.id not in forks: - assert trace.num_branches == 1, f"{trace.num_branches} branches" + assert trace.num_branches == 1, ( + f"{trace.num_branches} branches", + [f"{error.type}: {error.message[:500]}" for error in trace.errors], + ) def agent_seats(taskset: str, env_id: str = "") -> list[str]: @@ -145,6 +151,7 @@ def supports(runtime: str, feature: str) -> bool: "user": "SUPPORTS_USER_SWITCHING", "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", + "memory": "SUPPORTS_MEMORY_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -174,6 +181,8 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" + if "memory" in needs and not cls.SUPPORTS_MEMORY_RESTORE: + return f"{name} cannot continue from a memory snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py new file mode 100644 index 0000000000..441d3f9e95 --- /dev/null +++ b/tests/integration/test_checkpoint.py @@ -0,0 +1,419 @@ +"""A retry restores the box's newest memory snapshot (memory-restore-v1: the reply of one +model turn fails once and the box is snapshotted before the harness gets the error; the +retry continues a new box from that snapshot, and the harness program the box still runs +gets the reply and finishes). + +Every harness that continues from a memory snapshot (`Harness.SUPPORTS_MEMORY_RESTORE`) +runs it on prime, the one runtime with memory snapshots. A process the task's `setup` +started holds a nonce only in its memory: the reward reads it from the restored box, so +a box that booted from files alone would fail. Every case checks that the run leaks no +snapshot.""" + +import asyncio +import json +import pickle + +import pytest +from conftest import _eval_config +from grid import grid +from prime_sandboxes import AsyncSandboxClient +from subagent_restore import sub_agent_turns +from tick_restore import COMMAND, TURNS + +from verifiers.v1.clients import ModelContext +from verifiers.v1.dialects.chat import ChatDialect +from verifiers.v1.env import RunSlot +from verifiers.v1.restore import restore_entry, save_all +from verifiers.v1.runtimes import PrimeBackend, Snapshot +from verifiers.v1.utils.loaders import harness_class, load_environment + +RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} +NEEDS = ("memory",) +SHELLS = ["bash", "rlm"] +# rlm with its guidance on how to hand work to sub-agents. +SUB_AGENTS = {"id": "rlm", "delegation_prompt": True} + + +@pytest.fixture +def taken(monkeypatch) -> list[str]: + """The id of every snapshot the run's boxes take, in every attempt.""" + snapshots: list[str] = [] + snapshot = PrimeBackend.snapshot + + async def recording(self): + result = await snapshot(self) + snapshots.append(result.id) + return result + + monkeypatch.setattr(PrimeBackend, "snapshot", recording) + return snapshots + + +@pytest.fixture +async def command_model(): + """A scripted model for the bash harness: its k-th reply asks for the tick task's + command, and the reply after the last command says done. A box that asks again for + a turn gets the same reply. Yields a client config that points at it.""" + from aiohttp import web + + dialect = ChatDialect() + + async def answer(request: web.Request) -> web.StreamResponse: + body = await request.json() + turn = sum(m.get("role") == "assistant" for m in body["messages"]) + 1 + if turn <= TURNS: + call = { + "id": f"call_{turn}", + "type": "function", + "function": { + "name": "bash", + "arguments": json.dumps({"command": COMMAND}), + }, + } + message = {"role": "assistant", "content": None, "tool_calls": [call]} + else: + message = {"role": "assistant", "content": "done"} + raw = { + "id": f"chatcmpl-{turn}", + "object": "chat.completion", + "created": 0, + "model": str(body.get("model", "scripted")), + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if turn <= TURNS else "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + if not dialect.streaming(body): + return web.json_response(raw) + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + for event in dialect.stream_events(raw): + await response.write(event) + await response.write_eof() + return response + + app = web.Application() + app.router.add_post("/{tail:.*}", answer) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + try: + yield {"base_url": f"http://127.0.0.1:{port}/v1"} + finally: + await runner.cleanup() + + +async def platform_state(client: AsyncSandboxClient, snapshot_id: str) -> str | None: + """The snapshot's state on prime, or None once it is gone.""" + try: + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None + + +async def assert_no_leaks(taken: list[str]) -> None: + """Every snapshot the run took is deleted on the platform.""" + leaked = {} + async with AsyncSandboxClient() as client: + for snapshot_id in taken: + state = await platform_state(client, snapshot_id) + if state not in (None, "DELETING"): + leaked[snapshot_id] = state + assert leaked == {} + + +async def restored_trace( + run, harness, harness_runtime, *, fail_turn: int, **checkpoint +): + (episode,) = await run( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": fail_turn}}, + env={ + "agent": { + "retries": RETRIES, + "checkpoint": checkpoint, + } + }, + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + if not trace.errors: + # The harness handled the failed reply itself and finished its run: nothing + # failed, so nothing was restored. + pytest.skip(f"{harness.id} recovers from a failed model call by itself") + # The process `setup` started kept its memory across the restore. + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] + assert trace.attempt == 2 + assert trace.has_restored + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + # The harness program never restarted: one root. + assert sum(node.parent is None for node in trace.nodes) == 1 + # Every turn kept has its call; the failed turn's call stays, unlinked. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace + + +async def saved_rollout(config, ctx, ready=lambda trace: trace.num_turns >= 1): + """Run the config's one rollout until `ready(trace)`, checkpoint it with `save_all`, + and cancel it with everything it holds. Returns its slot and its restore point. By + default the rollout is past its first turn: the box holds work the trace recorded.""" + env = load_environment(config.env) + (task,) = list(env.taskset) + async with env.serving(): + (slot,) = env.slots(task) + running = asyncio.ensure_future(env.run_slot(slot, ctx)) + try: + async with asyncio.timeout(600): + while not (slot.traces and ready(slot.traces[0])): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.2) + (point,) = await save_all() + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + return slot, point + + +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): + """The box is snapshotted at every model turn, and the retry restores the snapshot + of the turn whose call failed. A harness with a shell fails at its second turn, so + the restored box holds the first turn's command and its harness goes on to the + second; a harness without one has a single turn, which fails. The calls that fail + again at that turn take no further snapshot.""" + shell = harness_class(harness.id).EXECUTES_CODE + fail_turn = 2 if shell else 1 + trace = await restored_trace( + run, harness, harness_runtime, fail_turn=fail_turn, interval=1 + ) + if shell: + assert trace.num_turns >= 2 + assert trace.metrics["copied"] == 1.0 + assert [c.id for c in trace.checkpoints if c.restored] == [taken[fail_turn - 1]] + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True +) +async def test_another_owner_continues_a_saved_rollout(taken, harness, harness_runtime): + """`save_all` checkpoints a running rollout for its owner. The rollout is then + cancelled with everything it holds, and its restore point goes through a pickle. A + second env, with a server, a tunnel and secrets of its own, continues the rollout + from the point: in a new box, with the harness program and the task's process + still running.""" + config = _eval_config( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": 99}}, + max_turns=12, + max_tokens=4096, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + slot, point = await saved_rollout(config, ctx) + turns_saved = len([node for node in point["nodes"] if node["sampled"]]) + point = pickle.loads(pickle.dumps(point)) + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point["id"] + assert trace.reward == 1.0 + assert trace.metrics["copied"] == 1.0, [ + (n.message.role, n.sampled, str(n.message.model_dump(exclude_none=True))[:600]) + for n in trace.nodes + ] + assert trace.has_restored and trace.attempt == 2 + assert trace.num_turns > turns_saved >= 1 + assert sum(node.parent is None for node in trace.nodes) == 1 + # The owner holds the snapshot: no rollout deleted it. + (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] + assert snapshot_id == restore_entry(point)["id"] + async with AsyncSandboxClient() as client: + assert (await client.get_checkpoint(snapshot_id)).state == "DURABLE" + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(SHELLS, needs=NEEDS), indirect=True +) +async def test_a_restore_point_that_leads_nowhere_starts_the_rollout_over( + taken, harness, harness_runtime +): + """A restore point whose snapshot the platform no longer restores costs the rollout + no retry: it starts over under the same trace id and finishes.""" + config = _eval_config( + "memory-restore-v1", + harness=harness, + runtime=harness_runtime, + taskset_overrides={"task": {"fail_turn": 99}}, + max_turns=12, + max_tokens=4096, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + + slot, point = await saved_rollout(config, ctx) + snapshot = restore_entry(point)["snapshot"] + snapshot_id, snapshot["id"] = snapshot["id"], "no-such-checkpoint" + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.id == point["id"] + assert trace.reward == 1.0 + assert trace.attempt == 3 + + async with AsyncSandboxClient() as client, asyncio.timeout(300): + while (await client.get_checkpoint(snapshot_id)).state != "DURABLE": + await asyncio.sleep(2) + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid([SUB_AGENTS], needs=NEEDS), indirect=True +) +async def test_another_owner_continues_running_sub_agents( + taken, harness, harness_runtime +): + """A rollout is saved while two sub-agents of its harness are at work (subagent- + restore). The second owner's box continues both: neither is started again, each + takes its next turns, and each reads its nonce from a process that only a box with + its memory still runs.""" + config = _eval_config( + "subagent-restore", + harness=harness, + runtime=harness_runtime, + max_turns=40, + max_tokens=4096, + rollout_timeout=600, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + slot, point = await saved_rollout( + config, ctx, ready=lambda trace: all(sub_agent_turns(trace).values()) + ) + saved = len(point["nodes"]) + point = pickle.loads(pickle.dumps(point)) + + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot(RunSlot(task, id=slot.id, restore=point), ctx) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.has_restored and trace.attempt == 2 + assert trace.reward == 1.0 + for name, turns in sub_agent_turns(trace).items(): + # At work when the box was saved, and at work again in the restored box. + assert [turn for turn in turns if turn < saved], name + assert [turn for turn in turns if turn >= saved], name + # The coordinator and each sub-agent started once: three conversations. + assert sum(node.parent is None for node in trace.nodes) == 3 + (snapshot_id,) = [c.id for c in trace.checkpoints if c.pinned] + assert await PrimeBackend.discard([Snapshot(id=snapshot_id)]) == [snapshot_id] + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_every_saved_turn_continues_to_the_same_result( + taken, harness, harness_runtime, command_model +): + """One rollout is saved at its first, second and third turn and then finishes. A + second owner continues each saved trace, and every continuation ends with the + messages of the first run: a restore from any turn loses no command and repeats + none. The model is scripted, so the turns after a restore are the same ones.""" + config = _eval_config( + "tick-restore", + harness=harness, + runtime=harness_runtime, + max_turns=12, + max_tokens=4096, + rollout_timeout=300, + client=command_model, + ) + ctx = ModelContext( + client=config.client, model=config.model, sampling=config.sampling + ) + env = load_environment(config.env) + (task,) = list(env.taskset) + points = [] + async with env.serving(): + (slot,) = env.slots(task) + running = asyncio.ensure_future(env.run_slot(slot, ctx)) + try: + for turn in range(1, TURNS): + async with asyncio.timeout(300): + while not (slot.traces and slot.traces[0].num_turns >= turn): + assert not running.done(), "the rollout ended before the save" + await asyncio.sleep(0.1) + (point,) = await save_all() + points.append(pickle.loads(pickle.dumps(point))) + episode = await running + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + (reference,) = episode.traces + assert reference.ok, [error.message for error in reference.errors] + assert reference.reward == 1.0 + + def messages(trace): + return [node.message.model_dump(exclude_none=True) for node in trace.nodes] + + async def continued(point): + second = load_environment(config.env) + (task,) = list(second.taskset) + async with second.serving(): + episode = await second.run_slot( + RunSlot(task, id=slot.id, restore=point), ctx + ) + (trace,) = episode.traces + return trace + + # The saves are at three different turns. + saved_turns = [sum(node["sampled"] for node in p["nodes"]) for p in points] + assert saved_turns == sorted(set(saved_turns)), saved_turns + for point, trace in zip(points, await asyncio.gather(*map(continued, points))): + assert trace.ok, [error.message for error in trace.errors] + assert trace.has_restored + assert trace.reward == 1.0 + assert sum(node.parent is None for node in trace.nodes) == 1 + assert messages(trace) == messages(reference) + snapshots = [Snapshot(id=restore_entry(point)["id"]) for point in points] + assert sorted(await PrimeBackend.discard(snapshots)) == sorted( + snapshot.id for snapshot in snapshots + ) + await assert_no_leaks(taken) diff --git a/tests/integration/test_relay.py b/tests/integration/test_relay.py new file mode 100644 index 0000000000..0d61c8cc1e --- /dev/null +++ b/tests/integration/test_relay.py @@ -0,0 +1,215 @@ +"""The interception relay on Prime: the server's tunnel is lost while a rollout's model +call is in flight: its registration is deleted and its frpc connection drops, so it +cannot log in again. The server opens a new tunnel, reroutes the box's relay to it, and +the rollout finishes as if nothing happened: the waiting model call goes through once, +on one branch. A relay that dies is named in the trace (`RelayError`), not the harness +that died on it. + +The model is scripted and holds its second reply while the test breaks something.""" + +import asyncio +import json +import os +import signal +from typing import Self + +import pytest +from aiohttp import web + +import verifiers.v1 as vf +from verifiers.v1.dialects.chat import ChatDialect +from verifiers.v1.harnesses.bash import BashHarnessConfig +from verifiers.v1.interception import InterceptionServer +from verifiers.v1.interception.relay import Relay +from verifiers.v1.utils.prime import load_prime_config + +pytestmark = pytest.mark.skipif( + not (os.environ.get("PRIME_API_KEY") or load_prime_config().get("api_key")), + reason="needs Prime credentials", +) + +DETECTION_SECONDS = 180 +"""How long the tunnel's health probe may take to declare the lost tunnel gone, plus the +time to open a new one.""" + + +def _tool_call(command: str, n: int) -> dict: + call = {"name": "bash", "arguments": json.dumps({"command": command})} + return { + "role": "assistant", + "content": "", + "tool_calls": [{"id": f"call_{n}", "type": "function", "function": call}], + } + + +SCRIPT = [ + _tool_call("echo before-the-tunnel-loss > /tmp/note", 0), + _tool_call("cat /tmp/note", 1), + {"role": "assistant", "content": "done"}, +] + + +class HeldModel: + """A chat-completions model that plays `SCRIPT`, one reply per turn, and holds the + reply to turn 1 until `release` is set. It counts the requests per turn.""" + + def __init__(self) -> None: + self.requests: dict[int, int] = {} + self.held = asyncio.Event() + self.release = asyncio.Event() + self.url = "" + self.runner: web.AppRunner | None = None + + async def answer(self, request: web.Request) -> web.StreamResponse: + body = await request.json() + turn = sum(1 for m in body["messages"] if m["role"] == "assistant") + self.requests[turn] = self.requests.get(turn, 0) + 1 + if turn == 1: + self.held.set() + await self.release.wait() + message = SCRIPT[min(turn, len(SCRIPT) - 1)] + raw = { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "created": 0, + "model": body.get("model", "scripted"), + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" + if "tool_calls" in message + else "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + dialect = ChatDialect() + if not dialect.streaming(body): + return web.json_response(raw) + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + for event in dialect.stream_events(raw): + await response.write(event) + await response.write_eof() + return response + + async def __aenter__(self) -> Self: + app = web.Application() + app.router.add_post("/{tail:.*}", self.answer) + self.runner = web.AppRunner(app) + await self.runner.setup() + site = web.TCPSite(self.runner, "127.0.0.1", 0) + await site.start() + self.url = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1" + return self + + async def __aexit__(self, *exc) -> None: + self.release.set() + assert self.runner is not None + await self.runner.cleanup() + + +async def drop_tunnel(url: str) -> None: + """Lose the tunnel that serves `url` for good: delete its registration, then end its + frpc connection, which then cannot log in again. A deleted registration alone keeps + serving while frpc stays connected.""" + from prime_tunnel import TunnelClient + + async with TunnelClient() as api: + tunnels = await api.list_tunnels(per_page=100) + (tunnel,) = [t for t in tunnels if t.url.rstrip("/") == url.rstrip("/")] + assert await api.delete_tunnel(tunnel.tunnel_id) + frpc = [pid for pid in _children() if f"{tunnel.tunnel_id}.toml" in _cmdline(pid)] + assert frpc, "no frpc process for the tunnel" + for pid in frpc: + os.kill(pid, signal.SIGKILL) + + +def _children() -> list[int]: + pids = [] + for entry in os.listdir("/proc"): + if not entry.isdigit(): + continue + try: + with open(f"/proc/{entry}/stat") as stat: + ppid = int(stat.read().rsplit(")", 1)[1].split()[1]) + except (OSError, ValueError, IndexError): + continue + if ppid == os.getpid(): + pids.append(int(entry)) + return pids + + +def _cmdline(pid: int) -> str: + try: + with open(f"/proc/{pid}/cmdline") as cmdline: + return cmdline.read() + except OSError: + return "" + + +async def test_rollout_survives_a_replaced_tunnel(): + async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: + agent = vf.make_agent( + vf.AgentConfig( + harness=BashHarnessConfig(id="bash"), + runtime=vf.PrimeConfig(image="python:3.11-slim", labels=["vf-ci"]), + model="scripted", + client=vf.EvalClientConfig(base_url=model.url), + ), + interception=server, + ) + run = asyncio.create_task(agent.run(vf.Task(vf.TaskData(idx=0, prompt="Go.")))) + await asyncio.wait_for(model.held.wait(), 600) + first_url = server.base_url + await drop_tunnel(first_url) + # Hold the reply until the server serves the rollout through a new tunnel. + async with asyncio.timeout(DETECTION_SECONDS): + while server.base_url == first_url: + await asyncio.sleep(1) + model.release.set() + trace = await asyncio.wait_for(run, 600) + + assert server.base_url != first_url + assert trace.ok, [error.message for error in trace.errors] + assert trace.errors == [] + tool_results = [str(m.content) for m in trace.messages if m.role == "tool"] + assert tool_results[-1].strip() == "before-the-tunnel-loss" + # The held call reached the model once: the relay's retry joined the request the + # server was already serving. + assert model.requests[1] == 1 + assert len(trace.branches) == 1 + + +async def test_a_dead_relay_is_named_in_the_trace(monkeypatch): + """The relay exits while a model call waits: the harness dies on the dropped + connection, and the trace records `RelayError`, not `HarnessError`.""" + relays: list[Relay] = [] + connect = Relay.connect + + async def recording(self, upstream, tokens): + relays.append(self) + return await connect(self, upstream, tokens) + + monkeypatch.setattr(Relay, "connect", recording) + async with HeldModel() as model, InterceptionServer(requires_tunnel=True) as server: + agent = vf.make_agent( + vf.AgentConfig( + harness=BashHarnessConfig(id="bash"), + runtime=vf.PrimeConfig(image="python:3.11-slim", labels=["vf-ci"]), + model="scripted", + client=vf.EvalClientConfig(base_url=model.url), + ), + interception=server, + ) + run = asyncio.create_task(agent.run(vf.Task(vf.TaskData(idx=0, prompt="Go.")))) + await asyncio.wait_for(model.held.wait(), 600) + (relay,) = relays + await relay.runtime.run(["sh", "-c", f'kill -9 "$(cat {relay.pid_path})"'], {}) + trace = await asyncio.wait_for(run, 600) + model.release.set() + + assert not trace.ok + assert [error.type for error in trace.errors] == ["RelayError"] + assert "the relay exited" in trace.errors[0].message diff --git a/tests/integration/test_shared_tool.py b/tests/integration/test_shared_tool.py index ddfd7f928c..e36dbaeaae 100644 --- a/tests/integration/test_shared_tool.py +++ b/tests/integration/test_shared_tool.py @@ -22,7 +22,9 @@ async def test_shared_tool(run, harness_runtime, tool_runtime): taskset_overrides={"tools": tool_runtime}, ) traces = [trace for episode in episodes for trace in episode.traces] - assert all(trace.num_branches == 1 for trace in traces) + assert all(trace.num_branches == 1 for trace in traces), [ + f"{error.type}: {error.message[:500]}" for t in traces for error in t.errors + ] assert len(traces) == 2 for trace in traces: assert trace.ok, [error.message for error in trace.errors] diff --git a/tests/unit/interception/test_relay.py b/tests/unit/interception/test_relay.py new file mode 100644 index 0000000000..9e27385428 --- /dev/null +++ b/tests/unit/interception/test_relay.py @@ -0,0 +1,551 @@ +"""The in-box relay forwards requests to its upstream with swapped tokens, streams +responses as they arrive, and on SIGHUP moves requests still waiting for their first +event to the new upstream on the same client connection.""" + +import asyncio +import http.client +import json +import os +import queue +import signal +import socket +import subprocess +import sys +import threading +import time +from collections.abc import Callable, Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from verifiers.v1.interception.relay import RELAY_SOURCE, relay_config +from verifiers.v1.interception.server import SERVER_HEADER + +TIMEOUT = 5 + + +class FakeUpstream: + """An HTTP server whose handler each test sets; it records every request.""" + + def __init__(self, port: int = 0) -> None: + self.requests: queue.Queue[dict] = queue.Queue() + self.respond: Callable[[BaseHTTPRequestHandler], None] = lambda h: None + self.stop = threading.Event() + upstream = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format, *args) -> None: + pass + + def handle_request(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + upstream.requests.put( + { + "method": self.command, + "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": self.rfile.read(length), + } + ) + try: + upstream.respond(self) + except OSError: + self.close_connection = True + + do_GET = do_POST = handle_request + + self.server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + self.server.daemon_threads = True + # The relay resets connections whose response it abandons. + self.server.handle_error = lambda request, address: None + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread( + target=self.server.serve_forever, args=(0.05,), daemon=True + ).start() + + def close(self) -> None: + self.stop.set() + self.server.shutdown() + self.server.server_close() + + +def reply(handler, status: int, body: dict) -> None: + data = json.dumps(body).encode() + handler.send_response(status) + handler.send_header(SERVER_HEADER, "1") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def start_sse(handler) -> None: + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + + +def chunk(handler, data: bytes) -> None: + handler.wfile.write(b"%x\r\n%s\r\n" % (len(data), data)) + handler.wfile.flush() + + +def end_sse(handler) -> None: + handler.wfile.write(b"0\r\n\r\n") + handler.wfile.flush() + + +def keepalives(upstream: FakeUpstream, before: bytes = b""): + """An SSE response that sends `before`, then only keepalives until it is cut.""" + + def respond(handler) -> None: + start_sse(handler) + if before: + chunk(handler, before) + while not upstream.stop.is_set(): + chunk(handler, b": keepalive\n") + upstream.stop.wait(0.05) + + return respond + + +def events(*data: bytes): + def respond(handler) -> None: + start_sse(handler) + for event in data: + chunk(handler, event) + end_sse(handler) + + return respond + + +class Relay: + def __init__( + self, + tmp_path: Path, + upstream: str, + tokens: dict[str, str], + source: str, + ) -> None: + script = tmp_path / "relay_server.py" + script.write_text(source) + self.config = tmp_path / "config.json" + self.write(upstream, tokens) + ready = tmp_path / "port" + self.failures = tmp_path / "failures" + self.stderr = (tmp_path / "relay.log").open("w+") + self.process = subprocess.Popen( + [sys.executable, str(script), "--config", str(self.config)] + + ["--port", "0", "--ready", str(ready)] + + ["--failures", str(self.failures)], + stderr=self.stderr, + ) + self.port = int(wait_for(lambda: ready.exists() and ready.read_text())) + + def write(self, upstream: str, tokens: dict[str, str]) -> None: + self.config.write_bytes(relay_config(upstream, tokens)) + + def reload(self, upstream: str, tokens: dict[str, str]) -> None: + self.write(upstream, tokens) + os.kill(self.process.pid, signal.SIGHUP) + + def connect(self) -> http.client.HTTPConnection: + return http.client.HTTPConnection("127.0.0.1", self.port, timeout=TIMEOUT) + + def close(self) -> str: + self.process.kill() + self.process.wait() + self.stderr.seek(0) + return self.stderr.read() + + +def wait_for(condition: Callable, timeout: float = TIMEOUT): + deadline = time.monotonic() + timeout + while not (value := condition()): + if time.monotonic() > deadline: + raise TimeoutError("condition not met") + time.sleep(0.01) + return value + + +def read_until(response: http.client.HTTPResponse, marker: bytes) -> bytes: + data = b"" + while marker not in data: + piece = response.read1(65536) + if not piece: + raise EOFError(data) + data += piece + return data + + +@pytest.fixture +def upstreams() -> Iterator[tuple[FakeUpstream, FakeUpstream]]: + a, b = FakeUpstream(), FakeUpstream() + yield a, b + a.close() + b.close() + + +@pytest.fixture +def make_relay(tmp_path) -> Iterator[Callable[..., Relay]]: + relays: list[Relay] = [] + + def make( + upstream: str, + tokens: dict[str, str] | None = None, + source: str = RELAY_SOURCE, + ) -> Relay: + relays.append(Relay(tmp_path, upstream, tokens or {}, source)) + return relays[-1] + + yield make + for relay in relays: + sys.stderr.write(relay.close()) + + +def sse_post(relay: Relay, token: str = "box") -> http.client.HTTPResponse: + connection = relay.connect() + connection.request( + "POST", + "/v1/chat/completions", + b'{"stream": true}', + {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + ) + return connection.getresponse() + + +def test_forwards_with_swapped_tokens(upstreams, make_relay): + upstream, _ = upstreams + upstream.respond = lambda h: reply(h, 401, {"error": "nope"}) + relay = make_relay(f"{upstream.url}/base/", {"box": "real", "key": "real-key"}) + connection = relay.connect() + for headers, expected in ( + ({"Authorization": "Bearer box"}, ("Bearer real", None)), + ({"x-api-key": "key"}, (None, "real-key")), + ( + {"Authorization": "Bearer other", "x-api-key": "other"}, + ("Bearer other", "other"), + ), + ): + connection.request("POST", "/v1/messages?beta=true", b'{"a": 1}', headers) + response = connection.getresponse() + assert response.status == 401 + assert response.getheader("Content-Type") == "application/json" + assert json.loads(response.read()) == {"error": "nope"} + seen = upstream.requests.get(timeout=TIMEOUT) + assert seen["method"] == "POST" + assert seen["path"] == "/base/v1/messages?beta=true" + assert seen["body"] == b'{"a": 1}' + assert seen["headers"]["host"] == upstream.url.removeprefix("http://") + auth = (seen["headers"].get("authorization"), seen["headers"].get("x-api-key")) + assert auth == expected + + +def test_streams_sse_as_it_arrives(upstreams, make_relay): + upstream, _ = upstreams + received = threading.Event() + + def respond(handler) -> None: + start_sse(handler) + chunk(handler, b"data: 1\n\n") + received.wait(TIMEOUT) + chunk(handler, b"data: 2\n\n") + end_sse(handler) + + upstream.respond = respond + response = sse_post(make_relay(upstream.url)) + assert response.status == 200 + assert response.getheader("Content-Type") == "text/event-stream" + assert read_until(response, b"data: 1\n\n") == b"data: 1\n\n" + received.set() + assert response.read() == b"data: 2\n\n" + + +def test_sighup_retries_waiting_request_on_new_upstream(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a) + b.respond = events(b"data: from-b\n\n", b"data: [DONE]\n\n") + relay = make_relay(a.url, {"box": "secret-a"}) + response = sse_post(relay) + assert response.status == 200 + read_until(response, b": keepalive\n") + assert ( + a.requests.get(timeout=TIMEOUT)["headers"]["authorization"] == "Bearer secret-a" + ) + relay.reload(b.url, {"box": "secret-b"}) + body = response.read() + assert body.endswith(b"data: from-b\n\ndata: [DONE]\n\n") + assert b"data:" not in body.partition(b"data: from-b")[0] + seen = b.requests.get(timeout=TIMEOUT) + assert seen["headers"]["authorization"] == "Bearer secret-b" + assert seen["headers"]["x-stainless-retry-count"] == "1" + assert seen["body"] == b'{"stream": true}' + + +def test_sighup_keeps_client_alive_until_new_upstream_is_up(upstreams, make_relay): + a, _ = upstreams + a.respond = keepalives(a) + with socket.socket() as free: + free.bind(("127.0.0.1", 0)) + port = free.getsockname()[1] + source = RELAY_SOURCE.replace( + "KEEPALIVE_SECONDS = 10.0", "KEEPALIVE_SECONDS = 0.05" + ) + relay = make_relay(a.url, source=source) + response = sse_post(relay) + read_until(response, b": keepalive\n") + relay.reload(f"http://127.0.0.1:{port}", {}) + read_until(response, b": relay keepalive\n") + b = FakeUpstream(port) + try: + b.respond = events(b"data: from-b\n\n") + assert response.read().endswith(b"data: from-b\n\n") + finally: + b.close() + + +def test_sighup_retry_with_other_status_sends_error_event(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a) + b.respond = lambda h: reply(h, 500, {"error": "boom"}) + relay = make_relay(a.url) + response = sse_post(relay) + read_until(response, b": keepalive\n") + relay.reload(b.url, {}) + assert response.read().endswith( + b'event: error\ndata: {"error": "relay: upstream changed status on retry"}\n\n' + ) + + +def test_sighup_after_data_closes_client_connection(upstreams, make_relay): + a, b = upstreams + a.respond = keepalives(a, before=b"data: partial\n\n") + relay = make_relay(a.url) + response = sse_post(relay) + read_until(response, b"data: partial\n\n") + relay.reload(b.url, {}) + with pytest.raises((http.client.HTTPException, OSError)): + while response.read1(65536): + pass + assert b.requests.empty() + + +def test_concurrent_requests_do_not_block_each_other(upstreams, make_relay): + upstream, _ = upstreams + both = threading.Barrier(2, timeout=TIMEOUT) + + def respond(handler) -> None: + both.wait() + events(b"data: done\n\n")(handler) + + upstream.respond = respond + relay = make_relay(upstream.url) + bodies: list[bytes] = [] + + def call() -> None: + bodies.append(sse_post(relay).read()) + + threads = [threading.Thread(target=call) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(TIMEOUT) + assert bodies == [b"data: done\n\n"] * 2 + + +def test_unreachable_upstream_answers_502_after_the_retry_window(make_relay): + source = RELAY_SOURCE.replace("RETRY_SECONDS = 5 * 60", "RETRY_SECONDS = 1") + relay = make_relay("http://127.0.0.1:1", source=source) + started = time.monotonic() + response = sse_post(relay) + assert response.status == 502 + assert "unreachable for 1s" in json.loads(response.read())["error"] + assert 1 <= time.monotonic() - started < 5 + # The host reads the reason back to name the relay, not the harness, in the trace. + assert "unreachable for 1s" in relay.failures.read_text() + + +def test_a_request_through_after_a_failure_clears_it(upstreams, make_relay): + a, _ = upstreams + a.respond = lambda handler: reply(handler, 200, {"ok": True}) + source = RELAY_SOURCE.replace("RETRY_SECONDS = 5 * 60", "RETRY_SECONDS = 1") + relay = make_relay("http://127.0.0.1:1", source=source) + assert sse_post(relay).status == 502 + relay.reload(a.url, {}) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + assert connection.getresponse().status == 200 + # The last line is the relay's current state: it forwards again. + assert wait_for(lambda: relay.failures.read_text().splitlines()[-1] == "ok") + + +def edge_page(status: int): + """A tunnel edge's own error page.""" + + def respond(handler) -> None: + body = b"Tunnel Not Found" + handler.send_response(status) + handler.send_header("Content-Type", "text/html") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + return respond + + +def test_edge_error_pages_are_retried(upstreams, make_relay): + a, _ = upstreams + answers = [edge_page(404), edge_page(502)] + + def respond(handler) -> None: + if answers: + answers.pop(0)(handler) + else: + reply(handler, 200, {"ok": True}) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request( + "POST", "/v1/chat/completions", b"{}", {"Authorization": "Bearer box"} + ) + response = connection.getresponse() + assert response.status == 200 + assert json.loads(response.read()) == {"ok": True} + retries = [ + a.requests.get(timeout=TIMEOUT)["headers"].get("x-stainless-retry-count") + for _ in range(3) + ] + # The edge's pages mean the server never saw the request, so the resends are + # not marked as retries: a repeated body is never answered from its replay cache. + assert retries == [None, None, None] + + +def test_interception_errors_reach_the_harness(upstreams, make_relay): + a, _ = upstreams + a.respond = lambda handler: reply(handler, 503, {"error": "provider down"}) + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + response = connection.getresponse() + assert response.status == 503 + assert json.loads(response.read()) == {"error": "provider down"} + assert a.requests.qsize() == 1 + + +def test_the_servers_own_404_reaches_the_harness(upstreams, make_relay): + """An unknown path is the server's answer, not a tunnel edge's page: the same + status and body type, told apart by the server's header.""" + a, _ = upstreams + + def respond(handler) -> None: + body = b"404: Not Found" + handler.send_response(404) + handler.send_header(SERVER_HEADER, "1") + handler.send_header("Content-Type", "text/plain") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/unknown", b"{}", {}) + response = connection.getresponse() + assert response.status == 404 + assert response.read() == b"404: Not Found" + assert a.requests.qsize() == 1 + + +def test_lost_upstream_is_retried_until_rerouted(upstreams, make_relay): + """The host replaces a lost tunnel: the request waits on the dead URL and goes + through once the relay is rerouted to the new one.""" + _, b = upstreams + b.respond = events(b"data: from-b\n\n") + relay = make_relay("http://127.0.0.1:1") + connection = relay.connect() + connection.request( + "POST", + "/v1/chat/completions", + b'{"stream": true}', + {"Authorization": "Bearer box"}, + ) + time.sleep(1) # the request is retrying against the dead upstream + relay.reload(b.url, {}) + response = connection.getresponse() + assert response.status == 200 + assert response.read().endswith(b"data: from-b\n\n") + # The dead upstream never saw the request: the first one to arrive is no retry. + assert "x-stainless-retry-count" not in b.requests.get(timeout=TIMEOUT)["headers"] + + +def test_body_cut_short_is_retried(upstreams, make_relay): + """An upstream that ends before its `Content-Length` (a tunnel that drops mid-body) + is retried while the harness has no body data yet.""" + a, _ = upstreams + body = json.dumps({"answer": "x" * 64}).encode() + attempts = [] + + def respond(handler) -> None: + attempts.append(True) + handler.send_response(200) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + if len(attempts) == 1: + handler.wfile.flush() + handler.close_connection = True + return + handler.wfile.write(body) + + a.respond = respond + relay = make_relay(a.url) + connection = relay.connect() + connection.request("POST", "/v1/chat/completions", b"{}", {}) + response = connection.getresponse() + assert response.status == 200 + assert response.read() == body + assert len(attempts) == 2 + + +class FakeRuntime: + """Answers the relay's failure check with a fixed stdout.""" + + def __init__(self, stdout: str) -> None: + self.stdout = stdout + + async def run(self, argv, env): + return SimpleNamespace(stdout=self.stdout, exit_code=0) + + +@pytest.mark.parametrize( + ("stdout", "expected"), + [ + ( + "upstream unreachable for 300s: refused\n", + "upstream unreachable for 300s: refused", + ), + ("the relay exited\n", "the relay exited"), + ("ok\n", None), + ("", None), + ], + ids=["gave-up", "exited", "recovered", "healthy"], +) +async def test_relay_failure_names_the_relay(monkeypatch, stdout, expected): + from verifiers.v1.errors import RelayError + from verifiers.v1.interception.relay import Relay + + monkeypatch.setattr(Relay, "_start", lambda self: asyncio.sleep(0)) + relay = Relay(FakeRuntime(stdout), "trace") + relay.upstream = "https://tunnel.example" + failure = await relay.failure() + if expected is None: + assert failure is None + else: + assert isinstance(failure, RelayError) and str(failure) == expected + await relay.stop() diff --git a/tests/unit/interception/test_tunnel.py b/tests/unit/interception/test_tunnel.py index 4c2ccc0f0f..aceb093fd7 100644 --- a/tests/unit/interception/test_tunnel.py +++ b/tests/unit/interception/test_tunnel.py @@ -1,5 +1,6 @@ """A prime tunnel is kept serving by the SDK through the service's health route. A tunnel -that can never work again retires its server: its rollouts fail and the pools skip it.""" +that can never work again retires its server: the pools skip it, its rollouts that reach +it through a relay move to a new tunnel, and the others fail.""" import asyncio import logging @@ -98,7 +99,9 @@ async def test_server_answers_the_health_route_on_a_connection_it_closes(): def test_gone_tunnel_fails_the_servers_rollouts_and_refuses_new_ones(): server = InterceptionServer() released = [] - session = SimpleNamespace(error=None, release=lambda: released.append(True)) + session = SimpleNamespace( + error=None, release=lambda: released.append(True), reroute=None + ) server.sessions["secret"] = session server._handle_tunnel_gone() @@ -109,6 +112,102 @@ def test_gone_tunnel_fails_the_servers_rollouts_and_refuses_new_ones(): server.register(SimpleNamespace()) +class ReplacingTunnel: + """Exposes the server at a new URL each time; `fail` makes the next expose fail.""" + + def __init__(self) -> None: + self.on_gone = None + self.urls = iter(["https://first.example", "https://second.example"]) + self.fail = False + + @asynccontextmanager + async def expose(self, port): + if self.fail: + raise TunnelError("no tunnel") + yield next(self.urls) + + +def movable_session(trace_id: str, moved: list) -> SimpleNamespace: + async def reroute(url: str) -> None: + moved.append((trace_id, url)) + + session = SimpleNamespace( + error=None, released=False, reroute=reroute, trace=SimpleNamespace(id=trace_id) + ) + session.release = lambda: setattr(session, "released", True) + return session + + +async def test_gone_tunnel_moves_movable_rollouts_to_a_new_tunnel(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.base_url = await server.stack.enter_async_context(server.tunnel.expose(0)) + moved: list = [] + movable = movable_session("movable", moved) + stuck = SimpleNamespace( + error=None, released=False, reroute=None, trace=SimpleNamespace(id="stuck") + ) + stuck.release = lambda: setattr(stuck, "released", True) + server.sessions.update(movable=movable, stuck=stuck) + + server._handle_tunnel_gone() + await server._replacing + + assert moved == [("movable", "https://second.example")] + assert server.base_url == "https://second.example" + assert movable.error is None and not movable.released + assert isinstance(stuck.error, TunnelError) and stuck.released + # The server still takes no new rollouts; the pools retire it once these leave. + assert not server.healthy + await server.stop() + + +async def test_tunnel_lost_during_a_replacement_is_replaced_again(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.tunnel.urls = iter( + ["https://first.example", "https://second.example", "https://third.example"] + ) + server.base_url = await server.stack.enter_async_context(server.tunnel.expose(0)) + moved: list = [] + + async def reroute(url: str) -> None: + moved.append(url) + if len(moved) == 1: + # The new tunnel is lost while the rollouts are rerouted to it. + server._handle_tunnel_gone() + + session = SimpleNamespace( + error=None, released=False, reroute=reroute, trace=SimpleNamespace(id="m") + ) + session.release = lambda: setattr(session, "released", True) + server.sessions["m"] = session + + server._handle_tunnel_gone() + await server._replacing + + assert moved == ["https://second.example", "https://third.example"] + assert server.base_url == "https://third.example" + assert session.error is None + await server.stop() + + +async def test_failed_replacement_fails_the_movable_rollouts(): + server = InterceptionServer() + server.tunnel = ReplacingTunnel() + server.tunnel.fail = True + moved: list = [] + movable = movable_session("movable", moved) + server.sessions["movable"] = movable + + server._handle_tunnel_gone() + await server._replacing + + assert moved == [] + assert isinstance(movable.error, TunnelError) and movable.released + await server.stop() + + async def test_elastic_pool_replaces_a_retired_server_and_frees_it(): async with ElasticInterceptionPool() as pool: retired = await pool._warm_task @@ -127,7 +226,9 @@ async def test_elastic_pool_replaces_a_retired_server_and_frees_it(): async def test_elastic_pool_keeps_a_retired_server_until_its_rollouts_leave(): async with ElasticInterceptionPool() as pool: retired = await pool._warm_task - retired.sessions["secret"] = SimpleNamespace(error=None, release=lambda: None) + retired.sessions["secret"] = SimpleNamespace( + error=None, release=lambda: None, reroute=None + ) retired._handle_tunnel_gone() replacement = await pool._server() diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..31becb317d 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -18,6 +18,7 @@ class FailingSegmentRollout: closed = Rollout.closed fail = Rollout.fail step = Rollout.step + _turn = Rollout._turn timeout = Rollout.timeout @@ -39,16 +40,20 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._opened = True run._closed = False run._failed = False + run._attempt = 0 run._failure = None run._borrowed_runtime = None run.runtime = None run._agent_time_remaining = time_remaining run._timeouts = RolloutTimeouts() run._harness_session = session + run._relay = None run.deployment = SimpleNamespace(run=lambda turn: turn) run._session = SimpleNamespace( request_interceptors=[], error=None, + aborted=None, + abort_event=asyncio.Event(), stopped=False, ) run.deadline_at = None @@ -99,3 +104,33 @@ async def turn(self, messages): assert trace.is_timeout and not trace.errors assert trace.stop_condition == "agent_timeout" + + +@pytest.mark.parametrize("outcome", ["keeps waiting", "dies"]) +async def test_request_hook_failure_fails_the_rollout(outcome): + """A request hook's failure is the task's verdict: it ends the segment at once and + fails the rollout, whatever the harness does meanwhile.""" + trace = make_trace() + error = vf.ProviderError("hook refused the turn", status_code=503) + cancelled = [] + + class WaitingSession: + async def turn(self, messages): + run._session.aborted = error + run._session.abort_event.set() + try: + if outcome == "keeps waiting": + await asyncio.sleep(60) + else: + raise RuntimeError("harness died") + except asyncio.CancelledError: + cancelled.append(True) + raise + + run = make_rollout(trace, WaitingSession(), time_remaining=5) + + await asyncio.wait_for(Interaction(run).turn("next"), timeout=1) + + assert [(e.type, e.stage) for e in trace.errors] == [("ProviderError", "agent")] + assert not trace.is_timeout + assert cancelled == ([True] if outcome == "keeps waiting" else []) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py new file mode 100644 index 0000000000..d3f0af20e3 --- /dev/null +++ b/tests/unit/test_checkpoint.py @@ -0,0 +1,555 @@ +"""Rollout checkpointing: when a rollout takes checkpoints, what `rewind` restores, and +what a restored attempt must not redo.""" + +import asyncio +import pickle +import time +from types import SimpleNamespace +from typing import Literal + +import verifiers.v1 as vf +import verifiers.v1.agent as agent_module +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import RolloutCheckpointConfig +from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.graph import MessageNode +from verifiers.v1.restore import open_restore, restore_entry, unsaveable +from verifiers.v1.rollout import Rollout, kept +from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession +from verifiers.v1.trace import ( + Checkpoint, + ModelCall, + TimeSpan, + TraceTask, +) +from verifiers.v1.types import ( + AssistantMessage, + ToolCall, + ToolMessage, + UserMessage, +) + + +def make_trace() -> vf.Trace: + trace = vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=TraceTask(type="TaskData", data=vf.TaskData(idx=0, prompt="q")), + ) + trace.nodes.append(MessageNode(parent=None, message=UserMessage(content="q"))) + trace.nodes.append( + MessageNode( + parent=0, + message=AssistantMessage( + content="", + tool_calls=[ToolCall(id="c1", name="bash", arguments="{}")], + ), + sampled=True, + ) + ) + trace.calls.append(ModelCall(node=1, time=TimeSpan(start=10.0, end=40.0))) + return trace + + +def test_rewind_restores_info_with_state(): + trace = make_trace() + checkpoint = Checkpoint( + id="ck", snapshot=vf.Snapshot(id="ck"), node=1, info={"setup": "nonce"} + ) + trace.checkpoints.append(checkpoint) + trace.info = {"setup": "nonce", "scoring_skipped": "stale"} + trace.rewind(checkpoint) + assert trace.info == {"setup": "nonce"} + + +def test_rewind_keeps_the_dropped_turns_calls(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", snapshot=vf.Snapshot(id="ck"), node=0) + trace.checkpoints.append(checkpoint) + trace.rewind(checkpoint) + assert len(trace.nodes) == 1 + assert [call.node for call in trace.calls] == [None] + + +def test_rewind_drops_the_later_checkpoints_at_the_same_node(): + trace = make_trace() + older = Checkpoint(id="older", node=1, snapshot=vf.Snapshot(id="older")) + target = Checkpoint(id="target", node=1, snapshot=vf.Snapshot(id="target")) + later = Checkpoint(id="later", node=1, snapshot=vf.Snapshot(id="later")) + trace.checkpoints.extend([older, target, later]) + assert trace.rewind(target) == [vf.Snapshot(id="later")] + assert [c.id for c in trace.checkpoints] == ["older", "target"] + + +def test_restored_span_counts_only_its_own_calls(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.split_agent_time() + assert trace.timing.agent.model.duration == 5.0 + assert trace.timing.agent.harness.duration == 15.0 + + +def test_agent_time_splits_off_the_checkpoint_hold(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.checkpoints += [ + Checkpoint(id="old", snapshot=vf.Snapshot(id="old"), captured=90.0, held=3.0), + Checkpoint(id="new", snapshot=vf.Snapshot(id="new"), captured=106.0, held=1.5), + ] + trace.split_agent_time() + assert trace.timing.agent.checkpoint.duration == 1.5 + assert trace.timing.agent.model.duration == 3.5 + assert trace.timing.agent.harness.duration == 15.0 + + +class SlowBackend: + """Stops the box for a snapshot until the test lets it run again; the snapshot + becomes durable when the test says so.""" + + def __init__(self) -> None: + self.durable = asyncio.Event() + self.stopped = asyncio.Event() + """Set once a snapshot has stopped the box.""" + self.running = asyncio.Event() + """A snapshot returns when this is set: the box runs again.""" + self.fail = False + self.taken = 0 + self.deleted: list[str] = [] + + async def snapshot(self) -> vf.Snapshot: + self.taken += 1 + self.stopped.set() + await self.running.wait() + return vf.Snapshot(id=f"ck{self.taken}") + + async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: + await self.durable.wait() + if self.fail: + raise RuntimeError("upload failed") + return 1024 + + async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: + ids = [snapshot.id for snapshot in snapshots] + self.deleted += ids + return ids + + +class FakeRelay: + upstream = "https://tunnel.example" + + def __init__(self) -> None: + self.state = {"dir": "/tmp/vf-relay/t", "url": "http://127.0.0.1:4000"} + self.rerouted: list[str] = [] + + async def reroute(self, upstream: str) -> None: + self.rerouted.append(upstream) + + +def make_rollout( + interval: int = 1, + keep: Literal["last"] | int = "last", + *, + running: bool = True, + stops: bool = False, +) -> tuple[Rollout, SlowBackend, RolloutSession]: + """A rollout that takes snapshots, with or without a harness segment that runs. + With `stops`, a snapshot keeps the box stopped until the test sets + `backend.running`.""" + backend = SlowBackend() + if not stops: + backend.running.set() + job = {"job_id": "j1"} if running else None + run = Rollout.__new__(Rollout) + run.trace = make_trace() # one turn so far + run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), trace=run.trace + ) + run._session = session + run._harness_session = SimpleNamespace( # type: ignore[assignment] + memory_state=lambda: None if job is None else {"job": job} + ) + run._relay = FakeRelay() # type: ignore[assignment] + run._secret = "box-secret" + run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) + run._durable = {} + run._captures = set() + run._closed = False + run._opened = True + run._memory = True + run.unsaveable = None + run.save_skipped = None + run._held = False + run._release_timer = None + run._capturing = asyncio.Lock() + run._pruning = asyncio.Lock() + run.deleted_snapshots = set() + run.rewound_snapshots = [] + run.restored_turns = None + return run, backend, session + + +async def test_checkpoint_interval_skips_turns(): + run, backend, _ = make_rollout(interval=2) + assert await run._checkpoint() is not None # turn 2: the 2nd + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + run.trace.nodes.append( + MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) + ) + assert await run._checkpoint() is None # turn 3 + assert backend.taken == 1 + + +async def test_the_restored_turn_is_not_snapshotted_again(): + run, backend, _ = make_rollout(interval=2) + run.restored_turns = run.trace.num_turns + assert await run._checkpoint() is None + assert backend.taken == 0 + + +async def test_checkpoint_becomes_durable_in_the_background(): + run, backend, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint() + assert checkpoint is not None and checkpoint.captured + assert checkpoint.time.end == 0 and checkpoint.bytes is None + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert checkpoint.time.end and checkpoint.bytes == 1024 + assert run.trace.checkpoints == [checkpoint] + + +async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): + run, backend, _ = make_rollout(interval=1) + await run._checkpoint() + backend.fail = True + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert run.trace.checkpoints == [] + + +async def test_settling_without_a_retry_stops_waiting(): + run, _, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint() + await asyncio.wait_for(run._settle_checkpoints(wait=False), timeout=1) + assert run._durable == {} + assert run.trace.checkpoints == [checkpoint] and checkpoint.time.end == 0 + + +async def test_a_closing_run_takes_no_checkpoint(): + run, backend, _ = make_rollout(interval=1) + capture = asyncio.create_task(run._checkpoint()) + run._closed = True + await run._settle_checkpoints(wait=True) + assert await capture is None + assert backend.taken == 0 and run.trace.checkpoints == [] + + +def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: + return [ + Checkpoint( + id=i, + snapshot=vf.Snapshot(id=i), + time=TimeSpan(start=1.0, end=2.0), + deleted=i in deleted, + ) + for i in ids + ] + + +def test_keep_policies(): + checkpoints = durable("a", "b", "c", "d", "e") + assert kept(checkpoints, "last") == {"e"} + assert kept(checkpoints, 1) == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 2) == {"a", "c", "e"} + assert kept(checkpoints, 3) == {"a", "d", "e"} + # Deleted and not yet durable snapshots are never kept. + pending = [ + *durable("a", "b", deleted=("a",)), + Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), + ] + assert kept(pending, 1) == {"b"} + + +async def test_a_durable_snapshot_prunes_the_older_ones(): + run, backend, _ = make_rollout(keep="last") + run.trace.checkpoints = durable("old1", "old2") + run.rewound_snapshots = [vf.Snapshot(id="rewound")] + backend.durable.set() + newest = await run._checkpoint() + await run._settle_checkpoints(wait=True) + assert sorted(backend.deleted) == ["old1", "old2", "rewound"] + assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] + assert run.deleted_snapshots == {"old1", "old2", "rewound"} + + +def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: + return vf.Agent( + vf.AgentConfig( + model="test-model", + harness={"id": harness}, + runtime=runtime, + checkpoint=RolloutCheckpointConfig(**checkpoint), + ) + ) + + +async def test_no_snapshot_at_turns_by_default(): + run, backend, _ = make_rollout() + run._checkpointing = RolloutCheckpointConfig() + assert await run._checkpoint() is None + assert backend.taken == 0 + + +async def test_a_call_asked_again_takes_no_second_snapshot(): + run, backend, _ = make_rollout(interval=1) + first = await run._checkpoint() + assert first is not None and backend.taken == 1 + # The same call comes again (the snapshot reset the box's connections). + assert await run._checkpoint() is None + assert backend.taken == 1 + + +def test_checkpointing_is_on_by_default(): + assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() + assert RolloutCheckpointConfig().interval is None + assert gated("bash", vf.PrimeConfig())._checkpointing() is not None + + +def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): + monkeypatch.setattr(agent_module, "_checkpointing_warned", set()) + for _ in range(2): + assert gated("bash", vf.SubprocessConfig())._checkpointing() is None + assert gated("terminus-2", vf.PrimeConfig())._checkpointing() is None + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert warnings == [ + ( + "rollout checkpointing is off: the 'subprocess' runtime takes no memory " + "snapshots; a failed rollout retries from the start" + ), + ( + "rollout checkpointing is off: harness 'terminus-2' cannot continue from " + "a memory snapshot; a failed rollout retries from the start" + ), + ] + + +async def test_a_snapshot_holds_every_reply_until_the_box_runs_again(): + run, backend, session = make_rollout(stops=True) + session.idempotent_requests = { + "retry:/v1/chat/completions:aa": IdempotentRequest( + binding=("/v1/chat/completions", b"aa"), + response=ReplayResponse(200, b"{}", "application/json"), + completed_at=time.monotonic(), + ), + "old": IdempotentRequest( + binding=("/v1/chat/completions", b"bb"), + response=ReplayResponse(200, b"{}", "application/json"), + completed_at=time.monotonic() - 3600, + ), + "open": IdempotentRequest(binding=("/v1/chat/completions", b"cc")), + } + capture = asyncio.create_task(run._checkpoint()) + await backend.stopped.wait() + # The box is stopped: nothing it asked for may be answered. + assert not session.flowing.is_set() + assert run._relay.rerouted == [] + backend.running.set() + checkpoint = await capture + assert session.flowing.is_set() + # The stop killed the box's connections: its relay opens new ones. + assert run._relay.rerouted == ["https://tunnel.example"] + assert checkpoint.snapshot == vf.Snapshot(id="ck1") + assert checkpoint.node == len(run.trace.nodes) - 1 + assert checkpoint.live == { + "program": {"job": {"job_id": "j1"}}, + "relay": run._relay.state, + "secret": "box-secret", + } + # Only a reply the box may not have received yet is kept for its restore. + assert list(checkpoint.replays) == ["retry:/v1/chat/completions:aa"] + assert not checkpoint.pinned + + +async def test_no_memory_snapshot_without_a_running_segment(): + run, backend, session = make_rollout(running=False) + assert await run._checkpoint() is None + assert backend.taken == 0 and session.flowing.is_set() + + +async def test_a_failed_memory_snapshot_lets_replies_flow_again(): + run, backend, session = make_rollout() + + async def failing(): + raise vf.SandboxError("pause refused") + + backend.snapshot = failing # type: ignore[method-assign] + assert await run._checkpoint() is None + assert session.flowing.is_set() and run.trace.checkpoints == [] + + +async def test_save_returns_a_trace_another_process_can_continue(): + run, backend, session = make_rollout() + session.pinned_tool_results = { + ("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned") + } + run.trace.info = {"nonce": "n"} + backend.running.set() + backend.durable.set() + saved = await run.save() + assert saved is not None + (checkpoint,) = run.trace.checkpoints + assert checkpoint.pinned + # The running rollout's own trace does not carry the saved fields. + assert checkpoint.restore is None + assert "restore" not in run.trace.model_dump()["checkpoints"][0] + + saved = pickle.loads(pickle.dumps(saved)) + assert saved["id"] == run.trace.id + assert restore_entry(saved)["snapshot"] == {"id": "ck1"} + task = vf.Task(vf.TaskData(idx=0, prompt="q")) + trace, restored = open_restore(saved, task) + assert restored.restore is None + assert trace.id == run.trace.id and trace.num_turns == run.trace.num_turns + assert restored is trace.checkpoints[0] and restored.pinned + assert restored.info == {"nonce": "n"} + assert restored.pins == session.pinned_tool_results + assert restored.live == checkpoint.live + # The restored trace rewinds to the point like a retry's. + assert trace.rewind(restored) == [] + + +async def test_save_returns_before_the_snapshot_is_durable(): + """The owner gets the restore point once the box runs again; the snapshot becomes + durable in the background, and the rollout's teardown waits for it.""" + run, backend, _ = make_rollout() + backend.running.set() + point = await run.save() + (checkpoint,) = run.trace.checkpoints + assert point is not None and not checkpoint.time.end + backend.durable.set() + await run._settle_checkpoints(wait=False) + assert checkpoint.time.end + + +async def test_a_held_save_answers_the_box_only_after_release(): + run, backend, session = make_rollout() + backend.running.set() + assert await run.save(hold=True) is not None + assert not session.flowing.is_set() + # A checkpoint of the run's own under the hold does not end it. + await run._checkpoint() + assert not session.flowing.is_set() + run.release() + assert session.flowing.is_set() and run._release_timer is None + + +async def test_a_release_does_not_answer_a_box_that_another_snapshot_stops(): + run, backend, session = make_rollout() + backend.running.set() + assert await run.save(hold=True) is not None + # A turn later the rollout takes a snapshot of its own, and the owner releases + # while the box is stopped for it. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + backend.running.clear() + backend.stopped.clear() + capture = asyncio.create_task(run._checkpoint()) + await backend.stopped.wait() + run.release() + assert not session.flowing.is_set() + backend.running.set() + assert await capture is not None + assert session.flowing.is_set() + + +async def test_a_cancelled_held_save_does_not_hold_the_box(): + run, backend, session = make_rollout() + backend.running.clear() + saving = asyncio.create_task(run.save(hold=True)) + await backend.stopped.wait() + saving.cancel() + assert (await asyncio.gather(saving, return_exceptions=True))[0] is not None + assert saving.cancelled() + assert session.flowing.is_set() + assert not run._held and run._release_timer is None + + +async def test_a_snapshot_that_waited_past_its_turn_is_not_taken(): + run, backend, session = make_rollout() + async with run._capturing: + capture = asyncio.create_task(run._checkpoint()) + await asyncio.sleep(0) + # The turn's reply lands on the trace while the snapshot waits its turn. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + assert await capture is None + assert backend.taken == 0 and session.flowing.is_set() + + +async def test_a_failed_held_save_does_not_hold_the_box(): + run, backend, session = make_rollout() + + async def failing(): + raise vf.SandboxError("pause refused") + + backend.snapshot = failing # type: ignore[method-assign] + assert await run.save(hold=True) is None + assert session.flowing.is_set() + assert run.save_skipped == "its memory snapshot failed" + + +async def test_a_rollout_without_memory_snapshots_saves_nothing(): + run, backend, _ = make_rollout() + run._memory, run.unsaveable = False, "the 'fake' runtime takes no memory snapshots" + assert await run.save() is None + assert backend.taken == 0 + assert run.save_skipped == "the 'fake' runtime takes no memory snapshots" + + +async def test_a_save_between_segments_names_its_reason(): + run, _, _ = make_rollout(running=False) + assert await run.save() is None + assert run.save_skipped == "its harness program is not running (setup or scoring)" + + +def test_unsaveable_names_what_rules_memory_snapshots_out(): + def reason(harness: str, runtime: vf.RuntimeConfig, **config) -> str | None: + return unsaveable( + vf.AgentConfig(harness={"id": harness}, runtime=runtime, **config) + ) + + assert reason("bash", vf.PrimeConfig()) is None + assert reason("rlm", vf.PrimeConfig()) is None + assert reason("bash", vf.SubprocessConfig()) == ( + "the 'subprocess' runtime takes no memory snapshots" + ) + assert reason("terminus-2", vf.PrimeConfig()) == ( + "harness 'terminus-2' cannot continue from a memory snapshot" + ) + assert reason("bash", vf.PrimeConfig(), checkpoint=None) == ( + "rollout checkpointing is off (agent.checkpoint)" + ) + + +async def test_an_owners_snapshot_outlives_the_rollout(): + """A pinned snapshot is not under the keep policy, is not deleted with the run's + own, and becomes durable even when the run settles without a retry to wait for.""" + run, backend, _ = make_rollout() + backend.running.set() + saving = asyncio.create_task(run.save()) + await asyncio.sleep(0) + while not run.trace.checkpoints: + await asyncio.sleep(0) + settling = asyncio.create_task(run._settle_checkpoints(wait=False)) + await asyncio.sleep(0) + assert not settling.done() + backend.durable.set() + await settling + assert await saving is not None + (pinned,) = run.trace.checkpoints + # A newer snapshot of the run's own, a turn later, does not prune the owner's. + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + newest = await run._checkpoint() + await run._settle_checkpoints(wait=True) + assert backend.deleted == [] + assert kept(run.trace.checkpoints, "last") == {newest.id} + assert not pinned.deleted diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 80cb35978e..a5dc143226 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -6,6 +6,7 @@ import verifiers.v1 as vf import verifiers.v1.agent as agent_module import verifiers.v1.utils.retries as retries_module +from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.env import Env, RunSlot RETRIES = vf.RetryConfig(max_retries=2) @@ -33,7 +34,11 @@ async def test_agent_retries_keep_the_trace_id(monkeypatch): minted: list[vf.Trace] = [] class FakeRollout: + runtime = None + def __init__(self, *, on_trace, trace_id, attempt, **_): + self.rewound_snapshots: list[str] = [] + self.deleted_snapshots: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) @@ -61,6 +66,57 @@ async def close(self) -> vf.Trace: assert [e.type for e in trace.errors] == ["SandboxError", "SandboxError"] +@pytest.mark.asyncio +async def test_a_restore_that_leads_nowhere_is_not_picked_again(monkeypatch): + """Attempt 1 takes a snapshot and fails. Attempt 2 restores it and fails before its + first new turn, with an error that no failed model call explains. The next retry + starts over instead of restoring the same snapshot again.""" + restores: list[str | None] = [] + + class FakeRollout: + runtime = None + + def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): + restores.append(None if restore is None else restore[1].id) + self.rewound_snapshots: list[vf.Snapshot] = [] + self.deleted_snapshots: set[str] = set() + self.restored_turns = None + if restore is None: + self.trace = make_trace(id=trace_id, attempt=attempt) + else: + self.trace = restore[0] + self.trace.attempt = attempt + self.restored_turns = self.trace.num_turns + + async def open(self) -> bool: + if self.trace.attempt == 1: + self.trace.checkpoints.append( + vf.Checkpoint( + id="c1", + snapshot=vf.Snapshot(id="c1"), + time=vf.TimeSpan(start=1.0, end=2.0), + ) + ) + if self.trace.attempt < 3: + self.trace.errors.append(FAILURE) + else: + self.trace.ok = True + return False + + async def close(self) -> vf.Trace: + return self.trace + + monkeypatch.setattr(agent_module, "Rollout", FakeRollout) + agent = vf.Agent(vf.AgentConfig(model="test-model", retries=RETRIES)) + monkeypatch.setattr(agent, "_rollout_params", lambda *args: {}) + monkeypatch.setattr(agent, "_checkpointing", lambda: RolloutCheckpointConfig()) + + trace = await agent.run(vf.Task(vf.TaskData(idx=0, prompt=None))) + + assert restores == [None, "c1", None] + assert trace.ok and trace.attempt == 3 + + @pytest.mark.asyncio async def test_episode_retries_keep_the_episode_id(): calls: list[tuple[str, int]] = [] diff --git a/uv.lock b/uv.lock index e2a1b7e7cb..80697900e2 100644 --- a/uv.lock +++ b/uv.lock @@ -3263,8 +3263,8 @@ toml = [ [[package]] name = "prime-sandboxes" -version = "0.5.9" -source = { registry = "https://pypi.org/simple" } +version = "0.5.11" +source = { git = "https://github.com/PrimeIntellect-ai/prime?subdirectory=packages%2Fprime-sandboxes&rev=fae6d0bc91717b95ff1b109d93d58a22e3408df2#fae6d0bc91717b95ff1b109d93d58a22e3408df2" } dependencies = [ { name = "aiofiles" }, { name = "certifi" }, @@ -3275,10 +3275,23 @@ dependencies = [ { name = "pyqwest" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8d/50/40e351b1fada75c8a166c3e482584912e660d7a143b5a1fa317c2038f2d0/prime_sandboxes-0.5.9.tar.gz", hash = "sha256:4aab8f2ca2dbd1f18cf356bb8c072baa29747cdee0305817d4d4eed7283c8cae", size = 173127, upload-time = "2026-11-10T01:03:33.691Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/77/888edf3076ef91d424f323aad4455e4a2c426cf1dde91728a46e19a029ac/prime_sandboxes-0.5.9-py3-none-any.whl", hash = "sha256:f66355ab2c8384d1ade329c82b9497165c490778031f927c00fefa6101f1536d", size = 93862, upload-time = "2026-11-10T01:03:31.814Z" }, -] + +[package.metadata] +requires-dist = [ + { name = "aiofiles", specifier = ">=23" }, + { name = "certifi" }, + { name = "connectrpc", specifier = ">=0.11.1,<0.12" }, + { name = "httpx", specifier = ">=0.25" }, + { name = "protobuf", specifier = ">=6.31.1" }, + { name = "pydantic", specifier = ">=2" }, + { name = "pyqwest", specifier = ">=0.6" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.13.1" }, + { name = "tenacity", specifier = ">=8" }, +] +provides-extras = ["dev"] [[package]] name = "prime-tunnel" diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 6f4811f79a..b0b7238ae2 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -65,6 +65,7 @@ HarnessError, InterceptionError, ProviderError, + RelayError, RolloutError, SandboxError, TaskError, @@ -95,6 +96,7 @@ Runtime, RuntimeInfo, RuntimeProcess, + Snapshot, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -119,6 +121,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, EvalRunInfo, EvalWorkInfo, @@ -208,6 +211,7 @@ "BaseConfig": "pydantic_config", "BindMount": "verifiers.v1.configs.runtime", "Branch": "verifiers.v1.trace", + "Checkpoint": "verifiers.v1.trace", "Client": "verifiers.v1.clients", "ClientConfig": "verifiers.v1.configs.client", "ComposeSpec": "verifiers.v1.runtimes.compose", @@ -276,6 +280,7 @@ "RuntimeConfig": "verifiers.v1.configs.runtime", "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", + "Snapshot": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplerSupport": "verifiers.v1.types", @@ -324,6 +329,7 @@ "TrainRunInfo": "verifiers.v1.trace", "TrainWorkInfo": "verifiers.v1.trace", "TunnelError": "verifiers.v1.errors", + "RelayError": "verifiers.v1.errors", "TurnTokens": "verifiers.v1.types", "Usage": "verifiers.v1.types", "UserMessage": "verifiers.v1.types", @@ -428,6 +434,7 @@ "StateT", "MessageNode", "Branch", + "Checkpoint", "TurnTokens", "SamplerSupport", "Timing", @@ -453,6 +460,7 @@ "TaskError", "InterceptionError", "TunnelError", + "RelayError", # clients "Client", "BaseClientConfig", @@ -483,6 +491,7 @@ "RuntimeProcess", "RuntimeConfig", "RuntimeInfo", + "Snapshot", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 4bed677ef5..af5a67fe4d 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -3,11 +3,14 @@ import asyncio import contextlib import json +import shlex +import time +import uuid from abc import abstractmethod from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeAlias, TypeVar, cast +from typing import Any, ClassVar, TypeAlias, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field @@ -16,6 +19,7 @@ from verifiers.v1.errors import HarnessError, InterceptionError from verifiers.v1.harness import Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess +from verifiers.v1.runtimes.base import current_program from verifiers.v1.semantic import ( ACP_SEMANTIC_EDGES_METADATA_KEY, SemanticEdgeSet, @@ -27,6 +31,9 @@ ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() MAX_PACKET_BYTES = 128 * 1024 * 1024 +ACP_ROOT = "/tmp/vf-acp" +"""Where a detached runner keeps its socket, log and request files, per rollout.""" +RUNNER_START_SECONDS = 120 _STOP_CONDITIONS = { "max_tokens": "max_output_tokens", "max_turn_requests": "max_turns", @@ -70,6 +77,8 @@ class ACPConfig: class ACPHarness(Harness[ConfigT]): """Harness backed by one live ACP process and native session per rollout.""" + SUPPORTS_MEMORY_RESTORE: ClassVar[bool] = True + async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script( ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"} @@ -237,6 +246,19 @@ def __init__( self._responses: dict[int, asyncio.Future] = {} self._request_id = 0 self._response_task: asyncio.Task | None = None + self._detached: list[str] | None = None + """The runner's argv once it runs as a job of the box (`memory`): each request + is then a job too, since no connection from the host survives a stopped box.""" + self._prompt: dict[str, Any] | None = None + """The reply file and the trace's call count of the prompt request under way.""" + + @property + def _dir(self) -> str: + return f"{ACP_ROOT}/{self.trace.id}" + + @property + def _running(self) -> bool: + return self._process is not None or self._detached is not None async def _start(self) -> None: self._stderr_tail.clear() @@ -245,6 +267,23 @@ async def _start(self) -> None: {**self.config.env, "UV_FROZEN": "false"}, activate=False, ) + if self.memory: + socket = f"{self._dir}/socket" + await self.runtime.run(["mkdir", "-p", self._dir], {}) + await self.runtime.run_background( + [*program, "--socket", socket], + self.config.env, + log=f"{self._dir}/runner.log", + ) + deadline = time.monotonic() + RUNNER_START_SECONDS + while (await self.runtime.run(["test", "-S", socket], {})).exit_code: + if time.monotonic() > deadline: + raise RuntimeError( + f"ACP runner did not start: {await self._runner_log()}" + ) + await asyncio.sleep(0.2) + self._detached = program + return process = await self.runtime.exec(program, self.config.env) self._process = process self._reader = _PacketReader(process.stdout) @@ -265,7 +304,71 @@ async def _read_responses(self, reader: _PacketReader) -> None: RuntimeError(f"ACP connection closed: {error}") ) + async def _runner_log(self) -> str: + with contextlib.suppress(Exception): + log = await self.runtime.run( + ["tail", "-c", "4000", f"{self._dir}/runner.log"], {} + ) + return log.stdout.strip() + return "" + + async def _call(self, operation: str, payload: dict[str, Any]) -> JsonObject: + """One request to the detached runner, as a job of the box.""" + assert self._detached is not None + name = uuid.uuid4().hex + request, reply = f"{self._dir}/{name}.request", f"{self._dir}/{name}.reply" + await self.runtime.write( + request, + json.dumps({"id": name, "operation": operation, **payload}).encode(), + ) + program = None + if operation == "prompt": + program = current_program.get() + self._prompt = {"reply": reply, "calls_before": len(self.trace.calls)} + try: + result = await self.runtime.run( + [*self._detached, "--call", f"{self._dir}/socket", request, reply], + self.config.env, + program=program, + ) + finally: + if operation == "prompt": + self._prompt = None + return await self._reply(result, reply) + + async def _reply(self, result: ProgramResult, reply: str) -> JsonObject: + if result.exit_code: + detail = (result.stderr or result.stdout).strip()[-1000:] + raise RuntimeError( + f"ACP runner request failed (exit {result.exit_code}): {detail}" + ) + response = json.loads(await self.runtime.read(reply)) + if not response.get("ok"): + self._stderr_tail[:] = (await self._runner_log()).encode() + return response + + def memory_state(self) -> dict[str, Any] | None: + state = super().memory_state() + if state is None or self._prompt is None or self._detached is None: + return None + return {**state, **self._prompt, "runner": list(self._detached)} + + async def _continue(self, state: dict[str, Any]) -> ProgramResult: + # The restored box runs the runner and the request's job; only their result + # is still to collect. + self._detached = list(state["runner"]) + async with self._lock: + try: + result = await self.runtime.attach(state["job"]) + response = await self._reply(result, state["reply"]) + except BaseException: + await run_shielded(self._stop(graceful=False)) + raise + return self._finish(response, state["calls_before"]) + async def _request(self, operation: str, **payload: Any) -> JsonObject: + if self._detached is not None: + return await self._call(operation, payload) process = self._process if process is None or self._response_task is None or self._response_task.done(): raise RuntimeError("ACP process is not running") @@ -289,7 +392,7 @@ async def steer(self, message: str, *, message_id: str | None = None) -> dict: raise RuntimeError("ACP session is closed") if not isinstance(message, str) or not message.strip(): raise ValueError("steering requires a nonempty user message") - if self._process is None: + if not self._running: return {"outcome": "promptRequired", "reason": "noRunningTurn"} response = await self._request("steer", message=message, message_id=message_id) if not response.get("ok"): @@ -344,16 +447,18 @@ async def _run(self, messages: Messages | None) -> ProgramResult: raise HarnessError( f"harness {self.harness.config.id!r} session is already closed" ) - if self._process is None: + if not self._running: await self._start() - assert self._process is not None - assert self._reader is not None calls_before = len(self.trace.calls) try: response = await self._request("prompt", config=config) except BaseException: await run_shielded(self._stop(graceful=False)) raise + return self._finish(response, calls_before) + + def _finish(self, response: JsonObject, calls_before: int) -> ProgramResult: + """Consume a prompt request's response as the segment's result.""" turn = _turn_result(response) self.trace.root_reply = turn.reply.strip() self.trace.info["acp_stop_reason"] = turn.stop_reason @@ -377,7 +482,33 @@ async def _run(self, messages: Messages | None) -> ProgramResult: _require_model_turn(self.trace, calls_before, result) return result + async def _stop_detached(self, *, graceful: bool) -> dict[str, Any]: + response_metadata: dict[str, Any] = {} + try: + if graceful: + with contextlib.suppress(BaseException): + response = await asyncio.wait_for(self._call("shutdown", {}), 30) + result = response.get("result") + if response.get("ok") and isinstance(result, dict): + metadata = result.get("response_metadata") + if isinstance(metadata, dict): + response_metadata = metadata + # Best effort: the box may be gone already. + pid = shlex.quote(f"{self._dir}/socket.pid") + with contextlib.suppress(BaseException): + await asyncio.wait_for( + self.runtime.run( + ["sh", "-c", f'kill "$(cat {pid})" 2>/dev/null; true'], {} + ), + 30, + ) + finally: + self._detached = None + return response_metadata + async def _stop(self, *, graceful: bool) -> dict[str, Any]: + if self._detached is not None: + return await self._stop_detached(graceful=graceful) process = self._process response_task = self._response_task stderr_task, self._stderr_task = self._stderr_task, None diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index b19942b5b9..2cf91e3357 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -361,27 +361,28 @@ def write_packet(stream: Any, value: dict) -> None: stream.flush() -async def serve_stream() -> None: - session = ACPSession() - reader = asyncio.StreamReader() - protocol = asyncio.StreamReaderProtocol(reader) - await asyncio.get_running_loop().connect_read_pipe( - lambda: protocol, sys.stdin.buffer - ) - tasks: set[asyncio.Task] = set() - prompt_lock = asyncio.Lock() +class Server: + """Serves the host's session requests, over whichever transport carries them.""" + + def __init__(self) -> None: + self.session = ACPSession() + self.tasks: set[asyncio.Task] = set() + self.prompt_lock = asyncio.Lock() - async def handle(request: dict) -> None: + async def handle(self, request: dict) -> dict: + session = self.session operation = request.get("operation") try: if operation == "prompt": - async with prompt_lock: + async with self.prompt_lock: result = asdict(await session.run(request["config"])) elif operation == "steer": result = await session.steer( request["message"], request.get("message_id") ) elif operation == "shutdown": + if self.tasks: + await asyncio.gather(*self.tasks, return_exceptions=True) result = {"response_metadata": await session.close()} else: raise ValueError(f"unknown ACP session operation: {operation!r}") @@ -395,27 +396,106 @@ async def handle(request: dict) -> None: if operation == "prompt": response["result"] = asdict(session.client.turn_result()) response["id"] = request.get("id") + return response + + def spawn(self, request: dict, reply) -> asyncio.Task: + """Handle `request` in a task of its own and hand its response to `reply`.""" + + async def run() -> None: + reply(await self.handle(request)) + + task = asyncio.create_task(run()) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + return task + + async def close(self) -> None: + for task in self.tasks: + task.cancel() + if self.tasks: + await asyncio.gather(*self.tasks, return_exceptions=True) + await self.session.close() + + +async def serve_stream() -> None: + server = Server() + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await asyncio.get_running_loop().connect_read_pipe( + lambda: protocol, sys.stdin.buffer + ) + + def reply(response: dict) -> None: write_packet(sys.stdout.buffer, response) try: while request := await read_packet(reader): if request.get("operation") == "shutdown": - if tasks: - await asyncio.gather(*tasks) - await handle(request) + reply(await server.handle(request)) break - task = asyncio.create_task(handle(request)) - tasks.add(task) - task.add_done_callback(tasks.discard) + server.spawn(request, reply) finally: - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - await session.close() + await server.close() + +async def serve_socket(path: str) -> None: + """Serve session requests on a Unix socket, one per connection, until a shutdown + request: the host's memory snapshots stop the box, which no connection from the + host survives, so each request comes from a job of the box (`call`) instead.""" + server = Server() + stopped = asyncio.Event() -async def main() -> None: + async def connected( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + try: + request = await read_packet(reader) + if request is None: + return + response = await server.handle(request) + data = json.dumps(response, ensure_ascii=False).encode() + writer.write(len(data).to_bytes(8, "big") + data) + await writer.drain() + if request.get("operation") == "shutdown": + stopped.set() + finally: + writer.close() + + with suppress(FileNotFoundError): + os.unlink(path) + listener = await asyncio.start_unix_server(connected, path, limit=MAX_PACKET_BYTES) + # A few bytes, once, before any request is served. + with open(f"{path}.pid", "w") as file: # noqa: ASYNC230 + file.write(f"{os.getpid()}\n") + try: + async with listener: + await stopped.wait() + finally: + await server.close() + + +def call(path: str, request_path: str, reply_path: str) -> None: + """Send the request in `request_path` to the server on the socket at `path` and + write its response to `reply_path`.""" + import socket + + with open(request_path, "rb") as file: + data = file.read() + with socket.socket(socket.AF_UNIX) as connection: + connection.connect(path) + connection.sendall(len(data).to_bytes(8, "big") + data) + stream = connection.makefile("rb") + size = int.from_bytes(stream.read(8), "big") + response = stream.read(size) + if len(response) != size: + raise EOFError("ACP runner closed the connection before its response") + staged = f"{reply_path}.tmp" + with open(staged, "wb") as file: + file.write(response) + os.replace(staged, reply_path) + + +async def main(socket_path: str | None) -> None: task = asyncio.current_task() loop = asyncio.get_running_loop() if task is not None: @@ -423,8 +503,15 @@ async def main() -> None: with suppress(NotImplementedError): loop.add_signal_handler(sig, task.cancel) with suppress(asyncio.CancelledError): - await serve_stream() + if socket_path is None: + await serve_stream() + else: + await serve_socket(socket_path) if __name__ == "__main__": - asyncio.run(main()) + arguments = sys.argv[1:] + if arguments[:1] == ["--call"]: + call(*arguments[1:4]) + else: + asyncio.run(main(arguments[1] if arguments[:1] == ["--socket"] else None)) diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 8abb971595..7b4a60037c 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -8,12 +8,13 @@ server; un-entered, each run brings its own.""" import asyncio +import logging import uuid from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Generic, Literal, Self, cast +from typing import Any, Generic, Literal, Self, cast from weakref import WeakValueDictionary from typing_extensions import TypeVar @@ -24,6 +25,7 @@ ) from verifiers.v1.configs.agent import ( AgentConfig, + RolloutCheckpointConfig, TimeoutConfig, agent_config_fields, resolve_agent, @@ -33,17 +35,19 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer +from verifiers.v1.restore import open_restore, restoring, unsaveable from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, backend_cls, provision_backend, runtime_is_local, ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -69,6 +73,9 @@ logger = get_logger(__name__) +# (harness id, runtime type) pairs already warned that they take no checkpoints. +_checkpointing_warned: set[tuple[str, str]] = set() + # Only nested runs in this scoring attempt participate in its rollback. _scoring_runs: ContextVar[list[tuple["_EpisodeAgent", Trace]] | None] = ContextVar( "scoring_runs", default=None @@ -659,6 +666,25 @@ def _interception_for( return self._server return None + def _checkpointing(self) -> RolloutCheckpointConfig | None: + """The checkpoint config when this harness on this runtime can take and restore + checkpoints; otherwise None, with a warning once per harness and runtime.""" + config = self.config.checkpoint + if config is None: + return None + reason = unsaveable(self.config, self.harness) + if reason is None: + return config + key = (self.harness.config.id, self.config.runtime.type) + if key not in _checkpointing_warned: + _checkpointing_warned.add(key) + logger.warning( + "rollout checkpointing is off: %s; a failed rollout retries from the " + "start", + reason, + ) + return None + def _check_resume_support(self) -> None: # Multi-turn capability is derived: a harness needs transcript replay, a # native resume implementation, or a rollout-scoped session implementation. @@ -685,6 +711,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: dict[str, Any] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -693,51 +720,184 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; every attempt's trace keeps - the run's id with its own `attempt`, and the final trace keeps earlier - attempts' errors. `on_score(trace, runtime)` runs after task scoring in - the selected verifier workspace, inside its timeout and retry boundary.""" + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry restores the failed attempt's newest + restorable checkpoint when it has one (`config.checkpoint`, on a runtime and a + harness that support it), keeping that trace and its turns up to there; a + checkpoint whose restore failed is skipped, and without one the retry starts + over. Every attempt's trace keeps the run's id with its own `attempt`, and the + final trace keeps every attempt's errors. `on_score(trace, runtime)` runs after task scoring in + the selected verifier workspace, inside its timeout and retry boundary. + `restore` continues a rollout another process saved (`save_all`) instead of + starting one: its trace, in a new box from its memory checkpoint; the episode's + saved trace (`RunSlot.restore`) is taken the same way.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries retry_state = RetryState(retry) - history: list = [] - trace_id = uuid.uuid4().hex - with scope(trace=short_id(trace_id), agent=self._log_name): - for attempt in range(retry_state.max_retries + 1): - trace = await self._run_once( - task, - runtime, - tools, - on_trace, - collect_artifacts, - on_score, - trace_id=trace_id, - attempt=attempt + 1, + checkpointing = self._checkpointing() if runtime is None else None + trace_id = uuid.uuid4().hex # one id for the run across its attempts + history: list[Error] = [] # errors of the attempts started over from + trace: Trace | None = None + target: Checkpoint | None = None # the checkpoint the next attempt restores + first_attempt = 0 # the attempts the trace made before this run + offered: str | None = None # the checkpoint of the owner's saved trace + pending = restoring.get() + if restore is None and pending: + restore = pending.pop() + if restore is not None: + if checkpointing is None: + logger.warning( + "Starting rollout %s over: this agent takes no checkpoints, so it " + "cannot continue from its saved trace", + restore.get("id"), ) - if attempt == retry_state.max_retries or trace.ok: - break - if runtime is not None: + else: + trace, target = open_restore(restore, task) + offered = target.id + trace_id = trace.id + first_attempt = trace.attempt + if on_trace is not None: + on_trace(trace) + snapshots: dict[str, Snapshot] = {} # every box snapshot any attempt took + deleted: set[str] = set() + failed_restores: set[str] = set() + run: Rollout | None = None + # The `finally` also covers a cancellation between attempts, when no rollout is + # open to delete them on abort. + budget = retry_state.max_retries + # Built once: every attempt runs the same harness instance, so per-run harness + # state (RLM's skill home) is the one a restored box holds. + params = self._rollout_params(task, runtime, dict(tools or {})) + attempt = 0 + with scope(trace=short_id(trace_id), agent=self._log_name): + try: + while True: + restoring_from = ( + (trace, target) + if trace is not None and target is not None + else None + ) + # This attempt's own errors decide — a restored trace still carries + # the earlier attempts'. + seen = ( + len(restoring_from[0].errors) + if restoring_from is not None + else 0 + ) + run = await self._run_once( + task, + runtime, + params, + on_trace, + collect_artifacts, + on_score, + trace_id=trace_id, + attempt=first_attempt + attempt + 1, + restore=restoring_from, + checkpoint=checkpointing, + ) + trace = run.trace + snapshots.update((s.id, s) for s in run.rewound_snapshots) + # An owner's snapshots (`save_all`) are the owner's to delete. + snapshots.update( + (c.snapshot.id, c.snapshot) + for c in trace.checkpoints + if not c.pinned + ) + deleted.update(run.deleted_snapshots) + if target is not None and ( + any(error.stage == "setup" for error in trace.errors[seen:]) + or ( + not trace.ok + and run.restored_turns is not None + and trace.num_turns <= run.restored_turns + ) + ): + # The restore led nowhere: the box never ran from it, or the + # attempt failed before its first new turn. The next retry + # steps back to an older snapshot, or starts over. + failed_restores.add(target.id) + if trace.ok: + break + if ( + target is not None + and target.id == offered + and target.id in failed_restores + ): + # The owner's saved trace is an offer, not an attempt of + # this run: without it the rollout would have started over, + # so it does that now and keeps its retries. + logger.warning( + "Starting rollout over: saved trace %s led nowhere - %s", + target.id, + clip(trace.errors[-1].message) if trace.errors else "", + ) + history.extend(trace.errors) + target = None + attempt += 1 + budget += 1 + continue + if attempt >= budget: + break + if runtime is not None: + logger.warning( + "Not retrying rollout on a borrowed box - its state is no " + "longer the task's start state" + ) + break + cause = retry_state.next_error(trace.errors[seen:]) + if cause is None: + break + target = next( + ( + c + for c in reversed(trace.checkpoints) + if c.time.end + and not c.deleted + and c.id not in failed_restores + ), + None, + ) + if target is None: + history.extend(trace.errors) + delay = backoff(attempt) logger.warning( - "Not retrying rollout on a borrowed box - its state is no " - "longer the task's start state" + "Retrying rollout in %s (attempt %d/%d) from %s - %s: %s", + format_time(delay), + attempt + 2, + budget + 1, + f"checkpoint {target.id}" + if target is not None + else "the start", + cause.type, + clip(cause.message), ) - break - cause = retry_state.next_error(trace.errors) - if cause is None: - break - history.extend(trace.errors) - delay = backoff(attempt) - logger.warning( - "Retrying rollout in %s (attempt %d/%d) - %s: %s", - format_time(delay), - attempt + 2, - retry_state.max_retries + 1, - cause.type, - clip(cause.message), - ) - await asyncio.sleep(delay) + await asyncio.sleep(delay) + attempt += 1 + finally: + # The run is final (or cancelled): delete every snapshot left. Every box + # is gone by now, so the platform refuses none of them. + drop = [ + snapshot + for snapshot_id, snapshot in sorted(snapshots.items()) + if snapshot_id not in deleted + ] + if drop and run is not None and run.runtime is not None: + try: + run.record_deleted( + await run_shielded( + run.runtime.backend.discard_snapshots(drop) + ) + ) + except Exception as e: + logger.warning( + "Failed to discard snapshots - %s: %s", + type(e).__name__, + e, + exc_info=logger.isEnabledFor(logging.DEBUG), + ) + assert trace is not None and run is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -748,15 +908,16 @@ async def _run_once( self, task: Task, runtime: Runtime | None, - shared_tools: Mapping[str, SharedToolServer] | None, + params: dict[str, Any], on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None, *, trace_id: str, attempt: int, - ) -> Trace: - params = self._rollout_params(task, runtime, dict(shared_tools or {})) + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, + ) -> Rollout: if collect_artifacts: _check_artifact_runtime(params["runtime_config"]) run = Rollout( @@ -765,12 +926,17 @@ async def _run_once( collect_artifacts=collect_artifacts, on_score=on_score, trace_id=trace_id, + restore=restore, + checkpoint=checkpoint, attempt=attempt, **params, ) try: if await run.open(): - await run.step() + if restore is not None: + await run.restore() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() @@ -780,7 +946,7 @@ async def _run_once( raise if trace.agent.runtime is not None: trace.agent.runtime.borrowed = runtime is not None - return trace + return run @asynccontextmanager async def interaction( diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 19d484563d..3c34215b99 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -1,10 +1,11 @@ """One env agent's config: who plays the seat, and its per-run caps.""" -from typing import Any +from typing import Any, Literal from pydantic import ( BaseModel, Field, + PositiveInt, SerializationInfo, SerializeAsAny, field_serializer, @@ -38,6 +39,29 @@ class TimeoutConfig(BaseConfig): isolated verifier setup and retries. Finalization has its own budget.""" +class RolloutCheckpointConfig(BaseConfig): + """Rollout checkpointing: snapshot the box's files and memory, so a retry or + another process continues the rollout from there instead of starting over. The + restored box continues the harness program itself and asks again for the model + call that was open. By default a rollout takes a snapshot only when its owner + saves it (`save_all`); `interval` adds snapshots the rollout takes by itself. A + snapshot stops the box for a few seconds and resets its open connections to the + outside. It needs a runtime that takes memory snapshots + (`SUPPORTS_MEMORY_SNAPSHOTS`), a harness that runs detached from the host + (`Harness.SUPPORTS_MEMORY_RESTORE`) and a rollout without tool servers or services + of its own. Elsewhere a retry starts over. Every snapshot is deleted once the run + is final.""" + + interval: PositiveInt | None = None + """Snapshot at every n-th model turn (the n-th, the 2n-th, ...), for failures + that leave no box to snapshot (the box died, the harness program crashed): a retry + then loses fewer than n turns instead of starting over. None takes none.""" + keep: Literal["last"] | PositiveInt = "last" + """Snapshots kept while the rollout runs: the newest (`"last"`), or every k-th plus + the newest (`k`; 1 keeps every one). A failed restore falls back to an older kept + one.""" + + class AgentConfig(BaseConfig): harness: SerializeAsAny[HarnessConfig] | None = None """The agent's program (None = the taskset's default harness).""" @@ -75,6 +99,12 @@ class AgentConfig(BaseConfig): timeout: TimeoutConfig = TimeoutConfig() retries: RetryConfig = RetryConfig() + checkpoint: RolloutCheckpointConfig | None = Field( + default_factory=RolloutCheckpointConfig + ) + """Rollout checkpointing, on by default: the rollout can be snapshotted, so its + owner can save it and a retry restores its newest snapshot instead of starting + over (see `RolloutCheckpointConfig`). None turns it off.""" network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 2ac1fcdc4e..de784d0a36 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -214,12 +214,19 @@ class PrimeConfig(NetworkPolicyConfig): checkpoint: str | None = None """A DURABLE filesystem checkpoint to start from instead of ``image`` (see the SDK's ``checkpoint`` and ``wait_for_checkpoint``).""" + restore_memory: bool = False + """With ``checkpoint``: continue the processes of the paused sandbox that the checkpoint + holds the memory of, instead of booting from its disk.""" idle_timeout: float | None = 3600 """Seconds of inactivity before the sandbox self-deletes (None disables).""" creates_per_min: int | None = None """Pace sandbox creation to this many per minute, enforced run-wide across every process of the run (None/<= 0 disables it). (Tunnel creation is limited separately — see interception.tunnel.prime.tunnel_limiter.)""" + snapshots_per_min: int | None = None + """Pace memory snapshots (a pause and a resume each) to this many per minute, + run-wide like ``creates_per_min`` (None/<= 0 disables it). An owner that saves + every running rollout at once (`save_all`) then spreads its pauses out.""" mounts: dict[str, PrimeVolume] = Field(default_factory=dict) """Guest paths mapped to Prime volumes, created if missing before the VM. diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1c58f9af84..11d7591d81 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import ( + Any, Generic, TypeVar, ) @@ -30,6 +31,7 @@ requires_tunnel, ) from verifiers.v1.mcp import SharedToolServer, serve_shared +from verifiers.v1.restore import restoring from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task from verifiers.v1.trace import Error, Trace, TraceTask @@ -67,6 +69,9 @@ class RunSlot: episode: Episode | None = None done: bool = False started: float | None = None + restore: dict[str, Any] | None = None + """A rollout another process saved (`save_all`): the episode's first attempt + continues it instead of starting one.""" @classmethod def finished(cls, episode: Episode) -> "RunSlot": @@ -246,6 +251,7 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, + restore: dict[str, Any] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -273,7 +279,11 @@ async def run_episode( async with boundary(EnvError, f"{type(self).__name__}.setup()"): await self.setup(agents) async with boundary(EnvError, f"{type(self).__name__}.run()"): - await self.run(task, agents) + token = restoring.set([restore] if restore is not None else None) + try: + await self.run(task, agents) + finally: + restoring.reset(token) if not episode.traces: raise ValueError( f"{type(self).__name__}.run() ran no agent — every " @@ -370,6 +380,8 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, + # A retry of the episode starts over. + restore=slot.restore if attempts == 1 else None, ) started: float | None = None diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index d581748e45..6ff8959ec9 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -78,6 +78,11 @@ class TunnelError(InterceptionError): or was lost.""" +class RelayError(InterceptionError): + """The relay in the box between the harness and the host interception server failed: + it did not start, gave up on a request, or exited.""" + + @contextlib.asynccontextmanager async def boundary(error_cls: type[RolloutError], what: str) -> AsyncIterator[None]: """Run a framework→code boundary, attributing any error escaping it to `error_cls`. An diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 907089a388..dd4a3d56b8 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -4,7 +4,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import DialectName, HarnessConfig, skill_destination @@ -15,7 +15,8 @@ SandboxError, boundary, ) -from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.runtimes import Program, ProgramResult, Runtime +from verifiers.v1.runtimes.base import current_program from verifiers.v1.task import TaskData from verifiers.v1.types import Message, Messages from verifiers.v1.utils.decorators import ( @@ -68,6 +69,11 @@ class Harness(ABC, Generic[ConfigT]): leaks host state (auth, config, processes) both ways. Only the minimal in-house loops (`bash`, `null`) override to False.""" + SUPPORTS_MEMORY_RESTORE: ClassVar[bool] = False + """Whether a box restored from a memory snapshot continues this harness: its + program runs as a job of the box that the host only waits for (`Runtime.run(..., + program=)`), and it reaches the host only through the endpoint it was given.""" + def __init__(self, config: ConfigT) -> None: self.config = config @@ -364,6 +370,13 @@ def __init__( self.data = data self.tool_interception_url = tool_interception_url self._closed = False + self.program = Program() + """The main program's job while a segment runs (`Runtime.run(..., program=)`).""" + self.memory = False + """Whether the rollout takes memory snapshots: the session then keeps no + connection into the box that a stopped box would break.""" + self._attached: dict[str, Any] | None = None + """The state of the segment the restored box still runs (`reattach`).""" async def steer(self, message: str, *, message_id: str | None = None) -> dict: """Deliver a user message during a running turn, if supported by the harness.""" @@ -377,10 +390,34 @@ async def turn(self, messages: Messages | None = None) -> None: raise HarnessError( f"harness {self.harness.config.id!r} session is already closed" ) - async with boundary(HarnessError, f"harness {self.harness.config.id!r}"): - result = await self._run(messages) + attached, self._attached = self._attached, None + token = current_program.set(self.program) + try: + async with boundary(HarnessError, f"harness {self.harness.config.id!r}"): + if attached is not None: + result = await self._continue(attached) + else: + result = await self._run(messages) + finally: + current_program.reset(token) await self.harness._check_result(self.trace, self.runtime, result) + def memory_state(self) -> dict[str, Any] | None: + """What `reattach` needs to continue the running segment in a box restored from + a memory snapshot taken now, or None when no segment runs.""" + if self.program.job is None: + return None + return {"job": dict(self.program.job)} + + def reattach(self, state: dict[str, Any]) -> None: + """Prepare this new session for a box restored from a memory snapshot: its + first `turn` waits for the segment the box still runs instead of starting + one. `state` is what `memory_state` returned when the snapshot was taken.""" + self._attached = state + + async def _continue(self, state: dict[str, Any]) -> ProgramResult: + return await self.runtime.attach(state["job"]) + async def _run(self, messages: Messages | None) -> ProgramResult: if messages is None: kwargs = ( diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 8f6d852144..b9f56996b5 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,6 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_MEMORY_RESTORE = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 503429ddb6..99e93a6d06 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,6 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_MEMORY_RESTORE = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/utils/launch.py b/verifiers/v1/harnesses/utils/launch.py index 786fb67fb3..55d03a287c 100644 --- a/verifiers/v1/harnesses/utils/launch.py +++ b/verifiers/v1/harnesses/utils/launch.py @@ -11,6 +11,7 @@ from verifiers.v1.dialects.chat import message_to_wire from verifiers.v1.harnesses.utils import compaction, core, mcp from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.runtimes.base import current_program from verifiers.v1.trace import Trace from verifiers.v1.types import Messages from verifiers.v1.utils import network @@ -104,5 +105,7 @@ async def launch_chat_program( source, config.resolved_env, activate=activate ) return await runtime.run( - [*program, *args], env if env is not None else {**config.resolved_env} + [*program, *args], + env if env is not None else {**config.resolved_env}, + program=current_program.get(), ) diff --git a/verifiers/v1/interception/relay/__init__.py b/verifiers/v1/interception/relay/__init__.py new file mode 100644 index 0000000000..6dd2abf841 --- /dev/null +++ b/verifiers/v1/interception/relay/__init__.py @@ -0,0 +1,192 @@ +"""Host side of the in-box relay (`server.py`). + +A harness in a remote box reaches the interception server through a relay that runs in +its box, at `127.0.0.1`. The relay forwards every request to its current upstream and +swaps the harness's bearer tokens for the upstream's, so the host can move the harness +to a new URL or new secrets without the harness noticing: `Relay.reroute`. + +Each rollout runs its own relay, under `RELAY_ROOT/` and on a port of its own, so +rollouts that share a box never share one. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import shlex +import time +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +from verifiers.v1.errors import RelayError, SandboxError, TunnelError + +if TYPE_CHECKING: + from verifiers.v1.runtimes.runtime import Runtime + +RELAY_SOURCE = (Path(__file__).resolve().parent / "server.py").read_text() +RELAY_ROOT = "/tmp/vf-relay" +RELAY_START_SECONDS = 60 +"""How long a launched relay gets to accept connections.""" + + +def relay_config(upstream: str, tokens: dict[str, str]) -> bytes: + """A relay's config file: the upstream base URL, and the harness's tokens mapped to + the upstream's.""" + return json.dumps({"upstream": upstream, "tokens": tokens}).encode() + + +class Relay: + """A rollout's relay in its box. It launches when created, before its upstream is + known, so its start overlaps the harness setup once its program is installed + (`installed`); `connect` points it at the interception server, `reroute` moves it, + `stop` ends it.""" + + def __init__( + self, runtime: Runtime, name: str, *, running: dict[str, str] | None = None + ) -> None: + """`running` is the `state` of a relay the box already runs (a box restored + from a memory snapshot): nothing is launched, and `connect` moves that relay.""" + self.runtime = runtime + self.dir = f"{RELAY_ROOT}/{name}" if running is None else running["dir"] + self.url = "" if running is None else running["url"] + """The base URL the harness reaches the interception server at, once launched.""" + self.upstream = "" + self.tokens: dict[str, str] = {} + self._connected = asyncio.Event() + self._installed = asyncio.Event() + if running is not None: + self._installed.set() + self._launch = asyncio.create_task( + self._start() if running is None else asyncio.sleep(0) + ) + + @property + def state(self) -> dict[str, str]: + """What a later `Relay(..., running=)` needs to take this relay over in a box + restored from a memory snapshot.""" + return {"dir": self.dir, "url": self.url} + + @property + def config_path(self) -> str: + return f"{self.dir}/config.json" + + @property + def pid_path(self) -> str: + return f"{self.dir}/pid" + + @property + def failures_path(self) -> str: + """Where the relay appends the reason for each request it gives up on.""" + return f"{self.dir}/failures" + + async def _start(self) -> None: + # Each launch reads its port from a file of its own: a relay an earlier attempt + # left in the box can never answer for this one. + ready = f"{self.dir}/port-{uuid.uuid4().hex}" + # The gateway creates the directory. Nothing listens on port 9 (discard). + await self.runtime.write( + self.config_path, relay_config("http://127.0.0.1:9", {}) + ) + program = await self.runtime.prepare_uv_script(RELAY_SOURCE, activate=False) + self._installed.set() + args = ["--config", self.config_path, "--port", "0", "--ready", ready] + args += ["--failures", self.failures_path] + await self.runtime.run_background( + [*program, *args, "--pid", self.pid_path], {}, log=f"{self.dir}/relay.log" + ) + deadline = time.monotonic() + RELAY_START_SECONDS + while True: + port = (await self.runtime.run(["cat", ready], {})).stdout.strip() + if port.isdigit(): + self.url = f"http://127.0.0.1:{port}" + return + if time.monotonic() > deadline: + raise SandboxError( + f"the relay did not start within {RELAY_START_SECONDS}s" + ) + await asyncio.sleep(0.1) + + async def installed(self) -> None: + """Wait until the relay's program is in the box, or its launch failed. The + install puts `uv` in the box when it has none, and a harness setup that does + the same must not run at the same time: one would run the other's half-written + binary.""" + waiter = asyncio.create_task(self._installed.wait()) + try: + await asyncio.wait( + {waiter, self._launch}, return_when=asyncio.FIRST_COMPLETED + ) + finally: + waiter.cancel() + + async def connect(self, upstream: str, tokens: dict[str, str]) -> str: + """Point the relay at the interception server once it runs, and return the base + URL the harness reaches the server at.""" + try: + await self._launch + await self._configure(upstream, tokens) + finally: + self._connected.set() + return self.url + + async def wait_connected(self) -> None: + """Wait until `connect` finished; raise if the relay did not start.""" + await self._connected.wait() + if not self.upstream: + raise TunnelError("the relay did not start") + + async def reroute( + self, upstream: str, tokens: dict[str, str] | None = None + ) -> None: + """Move the relay to `upstream`, with `tokens` if the secrets changed. Its + requests in flight retry against the new upstream.""" + await self.wait_connected() + await self._configure(upstream, self.tokens if tokens is None else tokens) + + async def _configure(self, upstream: str, tokens: dict[str, str]) -> None: + # Written whole, then moved: the relay must never read a partial config. + staged = f"{self.config_path}.next" + await self.runtime.write(staged, relay_config(upstream, tokens)) + config, pid = shlex.quote(self.config_path), shlex.quote(self.pid_path) + reload = f'mv {shlex.quote(staged)} {config} && kill -HUP "$(cat {pid})"' + result = await self.runtime.run(["sh", "-c", reload], {}) + if result.exit_code: + raise SandboxError( + f"the relay did not reload: {result.stderr.strip()[-500:]}" + ) + self.upstream, self.tokens = upstream, tokens + + async def failure(self) -> RelayError | None: + """The relay's failure, if it gave up on a request or exited; for a harness that + died, so the trace names the relay rather than the harness.""" + if not self.upstream: + return None + failures, pid = shlex.quote(self.failures_path), shlex.quote(self.pid_path) + check = ( + f"tail -n 1 {failures} 2>/dev/null; " + f'kill -0 "$(cat {pid})" 2>/dev/null || echo "the relay exited"' + ) + with contextlib.suppress(Exception): + result = await self.runtime.run(["sh", "-c", check], {}) + # "ok": a request went through after the last failure. + reasons = [r for r in result.stdout.strip().splitlines() if r != "ok"] + if reasons: + return RelayError("; ".join(reasons)) + return None + + async def stop(self) -> None: + """End the relay and remove its files; best effort, the box may be gone.""" + self._connected.set() + if not self._launch.done(): + self._launch.cancel() + await asyncio.wait({self._launch}) + # Also after a failed or cancelled launch: the process may run already. + pid, directory = shlex.quote(self.pid_path), shlex.quote(self.dir) + # Best effort: the box may be gone already. + with contextlib.suppress(Exception): + await self.runtime.run( + ["sh", "-c", f'kill "$(cat {pid})" 2>/dev/null; rm -rf {directory}'], + {}, + ) diff --git a/verifiers/v1/interception/relay/server.py b/verifiers/v1/interception/relay/server.py new file mode 100644 index 0000000000..f90243e89e --- /dev/null +++ b/verifiers/v1/interception/relay/server.py @@ -0,0 +1,667 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["certifi"] +# /// +"""Loopback HTTP relay between a harness program in the box and the host's interception +server. + +The harness is launched against `http://127.0.0.1:` instead of the host's URL. The +relay forwards every request to the configured upstream and swaps the box's bearer +tokens for the upstream's. The upstream can change while the harness runs: the host +replaces a lost tunnel with a new URL. On SIGHUP the relay re-reads its config, aborts +every in-flight upstream request and retries each one against the new upstream, as +long as its client has seen nothing but the status, headers and SSE comments: a model +call still waiting for its first event. + +While the upstream cannot be reached (a connect fails, a request breaks before its +response, the tunnel edge answers with its own error page), the relay retries the +request for up to `RETRY_SECONDS`, so the harness does not see a tunnel that is +reconnecting or being replaced. A retry is marked as one (`x-stainless-retry-count`), +so the interception server joins it to the request it already serves. + +Usage: relay_server.py --config --port [--ready ] [--pid ] + [--failures ] + +The config file is JSON: `{"upstream": "https://host/base", "tokens": {"": +""}}`. Once listening, the relay writes its port to `--ready` and its +process id to `--pid`. Each request it gives up on appends its reason to `--failures`, +so the host can tell a failure of the relay from one of the harness. +""" + +import argparse +import contextlib +import http.client +import json +import os +import signal +import socket +import ssl +import struct +import sys +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +import certifi + +KEEPALIVE_SECONDS = 10.0 +"""Interval of the comments an SSE client gets while its request waits for a retry.""" +RETRY_SECONDS = 5 * 60 +"""How long a request is retried while the upstream cannot be reached: long enough for +the host to notice a lost tunnel, open a new one and reroute the relay.""" +EDGE_STATUSES = frozenset({404, 502, 503, 504}) +"""Statuses a tunnel edge answers with its own page when it cannot reach the upstream +("404 Tunnel Not Found", a bad gateway).""" +SERVER_HEADER = "x-vf-interception" +"""Header on every response of the interception server, its own 404 for an unknown path +included. A response with an edge status and without it never reached the server.""" +BACKOFF_SECONDS = 0.5 +BACKOFF_CAP_SECONDS = 10.0 +CONNECT_TIMEOUT_SECONDS = 10.0 +CHUNK_BYTES = 64 * 1024 +RETRY_COUNT_HEADER = "x-stainless-retry-count" +HOP_BY_HOP = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) +# The relay sends the whole body itself and lets http.client set Host for the upstream. +REQUEST_DROPPED = HOP_BY_HOP | {"host", "content-length", "expect"} +# Not terminated by a blank line, like the interception server's own keepalives: some +# SSE clients try to decode the empty event a blank line would dispatch. +KEEPALIVE = b": relay keepalive\n" +STATUS_CHANGED = "relay: upstream changed status on retry" +# certifi's roots, as the harness's own HTTP clients use: a task image may have no system +# CA store (no `ca-certificates`), and the stdlib then trusts no certificate at all. +SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) +SSL_CONTEXT.load_default_certs() + + +def log(message: str) -> None: + print(f"relay: {message}", file=sys.stderr, flush=True) + + +def error_event(message: str) -> bytes: + return b"event: error\ndata: " + json.dumps({"error": message}).encode() + b"\n\n" + + +class Config: + def __init__(self, upstream: str, tokens: dict[str, str]) -> None: + parts = urlsplit(upstream) + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError(f"upstream must be an http(s) URL: {upstream!r}") + if not isinstance(tokens, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in tokens.items() + ): + raise ValueError("tokens must map strings to strings") + self.scheme = parts.scheme + self.host = parts.hostname + self.port = parts.port + self.prefix = parts.path.rstrip("/") + self.tokens = tokens + port = f":{parts.port}" if parts.port else "" + self.origin = f"{parts.scheme}://{parts.hostname}{port}{self.prefix}" + + @classmethod + def load(cls, path: str) -> "Config": + with open(path) as file: + data = json.load(file) + return cls(data["upstream"], data.get("tokens") or {}) + + def connection(self) -> http.client.HTTPConnection: + if self.scheme == "https": + return http.client.HTTPSConnection( + self.host, + self.port, + timeout=CONNECT_TIMEOUT_SECONDS, + context=SSL_CONTEXT, + ) + return http.client.HTTPConnection( + self.host, self.port, timeout=CONNECT_TIMEOUT_SECONDS + ) + + def swap(self, name: str, value: str) -> str: + if name == "x-api-key": + return self.tokens.get(value, value) + scheme, _, token = value.partition(" ") + token = token.strip() + if scheme.lower() == "bearer" and token in self.tokens: + return f"{scheme} {self.tokens[token]}" + return value + + +class Upstream: + """One in-flight upstream attempt, abortable from another thread.""" + + def __init__(self) -> None: + self.lock = threading.Lock() + self.sock: socket.socket | None = None + self.aborted = False + + def attach(self, sock: socket.socket) -> None: + with self.lock: + self.sock = sock + aborted = self.aborted + if aborted: + shutdown(sock) + + def abort(self) -> None: + with self.lock: + self.aborted = True + sock = self.sock + if sock is not None: + shutdown(sock) + + +def shutdown(sock: socket.socket) -> None: + # Unlike `close`, `shutdown` wakes a read blocked in another thread. The base class + # method because `SSLSocket.shutdown` also drops the TLS state that read still uses. + with contextlib.suppress(OSError): + socket.socket.shutdown(sock, socket.SHUT_RDWR) + + +class Relay: + def __init__(self, path: str, failures: str | None = None) -> None: + self.path = path + self.failures = failures + self.failures_lock = threading.Lock() + self.failed = False + """Whether the last request outcome recorded in `failures` is a failure.""" + self.config = Config.load(path) + self.changed = threading.Condition() + self.upstreams: set[Upstream] = set() + + def begin(self) -> tuple[Upstream, Config]: + """Register an attempt on the current config, atomically with reading it, so a + reload aborts every attempt that still uses the config it replaces.""" + with self.changed: + upstream = Upstream() + self.upstreams.add(upstream) + return upstream, self.config + + def record_failure(self, reason: str) -> None: + self._record(reason.replace("\n", " "), failed=True) + + def record_success(self) -> None: + """After a failure, record that a request went through again: the failure is + no longer the relay's current state.""" + if self.failed: + self._record("ok", failed=False) + + def _record(self, line: str, *, failed: bool) -> None: + if self.failures is None: + return + with ( + self.failures_lock, + contextlib.suppress(OSError), + open(self.failures, "a") as file, + ): + file.write(line + "\n") + self.failed = failed + + def end(self, upstream: Upstream) -> None: + with self.changed: + self.upstreams.discard(upstream) + + def wait(self, config: Config, timeout: float) -> None: + with self.changed: + self.changed.wait_for(lambda: self.config is not config, timeout) + + def reload(self) -> None: + try: + config = Config.load(self.path) + except (OSError, ValueError, KeyError) as error: + # Aborting would only retry against the old upstream. + log(f"reload failed, keeping {self.config.origin}: {error}") + return + with self.changed: + self.config = config + aborted, self.upstreams = self.upstreams, set() + self.changed.notify_all() + for upstream in aborted: + upstream.abort() + log(f"reloaded: upstream {config.origin}, aborted {len(aborted)}") + + +class UpstreamError(Exception): + pass + + +def is_edge_error(response: http.client.HTTPResponse) -> bool: + """Whether `response` is the tunnel edge's own error page, not the upstream's answer.""" + return ( + response.status in EDGE_STATUSES and response.getheader(SERVER_HEADER) is None + ) + + +class ClientGone(Exception): + pass + + +class Scanner: + """Whether a response body forwarded anything but SSE comment lines and blank lines. + A client skips those, so a retry's body can still follow them.""" + + def __init__(self) -> None: + self.sse = False + self.data = False + self.line_start = True + + def feed(self, chunk: bytes) -> None: + if self.data or not chunk: + return + if not self.sse: + self.data = True + return + for byte in chunk: + if self.line_start: + if byte in b"\r\n": + continue + if byte != ord(":"): + self.data = True + return + self.line_start = False + elif byte in b"\r\n": + self.line_start = True + + def line_break(self) -> bytes: + """What ends a comment line that a broken upstream left unfinished.""" + return b"" if self.line_start else b"\n" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server: "Server" + + def __getattr__(self, name: str): + # Every method is relayed: `BaseHTTPRequestHandler` dispatches to `do_`. + if name.startswith("do_"): + return self.relay + raise AttributeError(name) + + def log_message(self, format: str, *args) -> None: + log(format % args) + + def relay(self) -> None: + try: + self.body = self.read_body() + except (OSError, ValueError): + self.send_error(400, "relay: unreadable request body") + return + target = self.path + if not target.startswith("/"): + parts = urlsplit(target) + target = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") + self.target = target + dropped = REQUEST_DROPPED | { + name.strip().lower() + for name in self.headers.get("Connection", "").split(",") + } + self.forwarded = [ + (name, value) + for name, value in self.headers.items() + if name.lower() not in dropped + ] + try: + self.retry_count = int(self.headers.get(RETRY_COUNT_HEADER, 0)) + except ValueError: + self.retry_count = 0 + self.retries = 0 + self.sent = 0 + """Attempts that may have reached the interception server: the retry count it + sees. An attempt that failed to connect or got the tunnel edge's own page did + not, so a request that a legitimate later turn repeats is never taken for a + retry and answered from the server's replay cache.""" + self.failing_since: float | None = None + self.delay = BACKOFF_SECONDS + self.sent_status: int | None = None + self.sent_length: int | None = None + self.chunked = False + self.scanner = Scanner() + self.write_lock = threading.Lock() + relay = self.server.relay + try: + while True: + try: + with self.keepalive(): + upstream, connection, response = self.open() + except UpstreamError as error: + log(f"{self.command} {self.label}: {error}") + self.fail(str(error)) + return + opened = time.monotonic() + try: + done = self.stream(upstream, response) + finally: + relay.end(upstream) + connection.close() + if done: + relay.record_success() + retried = f" after {self.retries} retries" if self.retries else "" + log(f"{self.command} {self.label} {self.sent_status}{retried}") + return + if self.scanner.data: + log(f"{self.command} {self.label}: aborted after data, closing") + self.reset() + return + if upstream.aborted: + self.retries += 1 + log(f"{self.command} {self.label}: aborted, retrying") + else: + if time.monotonic() - opened > BACKOFF_CAP_SECONDS: + # The response was up a while: this is a new outage. + self.failing_since = None + try: + self.unreachable(relay.config, "the response broke") + except UpstreamError as error: + self.fail(str(error)) + return + except ClientGone: + log(f"{self.command} {self.label}: client went away") + self.close_connection = True + + @property + def label(self) -> str: + return self.target.partition("?")[0] + + def read_body(self) -> bytes: + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size = int(self.rfile.readline(65537).split(b";")[0], 16) + if size == 0: + while self.rfile.readline(65537) not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline(65537) + length = int(self.headers.get("Content-Length") or 0) + return self.rfile.read(length) if length else b"" + + def upstream_headers(self, config: Config) -> dict[str, str]: + headers: dict[str, str] = {} + for name, value in self.forwarded: + lower = name.lower() + if lower in ("authorization", "x-api-key"): + value = config.swap(lower, value) + elif lower == RETRY_COUNT_HEADER and self.sent: + continue + headers[name] = f"{headers[name]}, {value}" if name in headers else value + if self.sent: + headers[RETRY_COUNT_HEADER] = str(self.retry_count + self.sent) + return headers + + def open( + self, + ) -> tuple[Upstream, http.client.HTTPConnection, http.client.HTTPResponse]: + relay = self.server.relay + while True: + upstream, config = relay.begin() + connection = config.connection() + try: + connection.connect() + assert connection.sock is not None + upstream.attach(connection.sock) + # Only the connect is bounded: a model call may wait long for its first + # byte. + connection.sock.settimeout(None) + connection.request( + self.command, + config.prefix + self.target, + self.body or None, + self.upstream_headers(config), + ) + self.sent += 1 + response = connection.getresponse() + except (OSError, http.client.HTTPException) as error: + relay.end(upstream) + connection.close() + if upstream.aborted: + self.retries += 1 + log(f"{self.command} {self.label}: aborted, retrying") + continue + self.unreachable(config, f"{error!r}") + continue + if is_edge_error(response): + # Not read: an edge that stalls its page must not block the retry. + self.sent -= 1 + relay.end(upstream) + connection.close() + self.unreachable(config, f"tunnel edge answered {response.status}") + continue + return upstream, connection, response + + def unreachable(self, config: Config, reason: str) -> None: + """Wait before the next attempt while the upstream cannot be reached: until the + config changes, at most `RETRY_SECONDS` after the first failure.""" + now = time.monotonic() + if self.failing_since is None: + self.failing_since = now + self.delay = BACKOFF_SECONDS + if now - self.failing_since >= RETRY_SECONDS: + raise UpstreamError( + f"upstream unreachable for {RETRY_SECONDS:g}s: {reason}" + ) + log(f"{config.origin} unreachable ({reason}), retry in {self.delay}s") + self.retries += 1 + self.server.relay.wait(config, self.delay) + self.delay = min(self.delay * 2, BACKOFF_CAP_SECONDS) + + @contextlib.contextmanager + def keepalive(self) -> Iterator[None]: + """Keep an SSE client's idle timer from firing while its request waits for a + retry.""" + if self.sent_status is None or not self.scanner.sse: + yield + return + stop = threading.Event() + + def tick() -> None: + while not stop.wait(KEEPALIVE_SECONDS): + try: + self.send(self.scanner.line_break() + KEEPALIVE) + except ClientGone: + return + + thread = threading.Thread(target=tick, daemon=True) + thread.start() + try: + yield + finally: + stop.set() + thread.join() + + def stream(self, upstream: Upstream, response: http.client.HTTPResponse) -> bool: + """Forward one upstream response; False when a reload aborted it.""" + if self.sent_status is None: + self.start(response) + elif response.status != self.sent_status: + log( + f"{self.command} {self.label}: retry answered {response.status}, " + f"sent {self.sent_status}" + ) + if self.scanner.sse: + self.send(self.scanner.line_break() + error_event(STATUS_CHANGED)) + self.finish_body() + else: + self.reset() + return True + elif self.sent_length is not None and response.length != self.sent_length: + self.reset() + return True + else: + self.send(self.scanner.line_break()) + while True: + try: + data = response.read1(CHUNK_BYTES) + except (OSError, http.client.HTTPException) as error: + if upstream.aborted: + return False + log(f"{self.command} {self.label}: upstream broke: {error!r}") + if not self.scanner.data: + # No body data went out: the retry can still answer. + return False + self.reset() + return True + # Data read after the abort is dropped: the retry sends it again. + if upstream.aborted: + return False + if not data: + if response.length: + # The upstream ended before its `Content-Length`. + log(f"{self.command} {self.label}: upstream body cut short") + if not self.scanner.data: + return False + self.reset() + return True + self.finish_body() + return True + self.send(data) + + def start(self, response: http.client.HTTPResponse) -> None: + status = response.status + content_type = response.getheader("Content-Type", "") + encoding = response.getheader("Content-Encoding", "identity") + self.scanner.sse = content_type.startswith("text/event-stream") and ( + encoding.lower() == "identity" + ) + dropped = HOP_BY_HOP | {"content-length"} + dropped |= { + name.strip().lower() + for name in (response.getheader("Connection") or "").split(",") + } + self.send_response_only(status, response.reason) + for name, value in response.getheaders(): + if name.lower() not in dropped: + self.send_header(name, value) + bodyless = self.command == "HEAD" or status in (204, 304) or status < 200 + if bodyless: + if length := response.getheader("Content-Length"): + self.send_header("Content-Length", length) + elif ( + response.length is not None + and not response.chunked + and not self.scanner.sse + ): + self.sent_length = response.length + self.send_header("Content-Length", str(response.length)) + elif self.request_version != "HTTP/1.0": + self.chunked = True + self.send_header("Transfer-Encoding", "chunked") + else: + self.close_connection = True + self.send_header("Connection", "close") + try: + self.end_headers() + except OSError as error: + raise ClientGone from error + self.sent_status = status + + def send(self, data: bytes) -> None: + if not data: + return + with self.write_lock: + self.scanner.feed(data) + if self.chunked: + data = b"%x\r\n%s\r\n" % (len(data), data) + try: + self.wfile.write(data) + self.wfile.flush() + except OSError as error: + raise ClientGone from error + + def finish_body(self) -> None: + if self.chunked: + with self.write_lock: + try: + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + except OSError as error: + raise ClientGone from error + + def fail(self, message: str) -> None: + self.server.relay.record_failure(message) + if self.sent_status is None: + body = json.dumps({"error": f"relay: {message}"}).encode() + self.send_response_only(502) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + try: + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + except OSError as error: + raise ClientGone from error + elif self.scanner.sse: + self.send(self.scanner.line_break() + error_event(f"relay: {message}")) + self.finish_body() + else: + self.reset() + + def reset(self) -> None: + """Break the client's connection so it cannot take a cut body for a whole one.""" + self.close_connection = True + with contextlib.suppress(OSError): + self.connection.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0) + ) + + +class Server(ThreadingHTTPServer): + daemon_threads = True + request_queue_size = 128 + + def __init__(self, address: tuple[str, int], relay: Relay) -> None: + self.relay = relay + super().__init__(address, Handler) + + +def write_file(path: str, text: str) -> None: + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + tmp = f"{path}.{os.getpid()}.tmp" + with open(tmp, "w") as file: + file.write(text) + os.replace(tmp, path) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.partition("\n")[0]) + parser.add_argument("--config", required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--ready") + parser.add_argument("--pid") + parser.add_argument("--failures") + args = parser.parse_args() + relay = Relay(args.config, args.failures) + # Blocked before any thread starts, so threads inherit the mask and only the + # `sigwait` below takes SIGHUP: a reload then never interrupts a handler. + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGHUP}) + server = Server(("127.0.0.1", args.port), relay) + port = server.server_address[1] + threading.Thread(target=server.serve_forever, daemon=True).start() + if args.pid: + write_file(args.pid, f"{os.getpid()}\n") + if args.ready: + write_file(args.ready, f"{port}\n") + log(f"listening on 127.0.0.1:{port}, upstream {relay.config.origin}") + while True: + signal.sigwait({signal.SIGHUP}) + relay.reload() + + +if __name__ == "__main__": + main() diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 10ef90aa6e..d7b48e2df6 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -32,6 +32,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass, field from functools import partial +from typing import NoReturn import httpx from aiohttp import web @@ -92,6 +93,10 @@ # Attempt counter the stainless-generated SDKs (OpenAI, Anthropic) send on every request: # 0 on the first attempt, incremented on each retry of the same request. RETRY_COUNT_HEADER = "x-stainless-retry-count" +SERVER_HEADER = "x-vf-interception" +"""Marks every response as this server's own, so a box's relay can tell an answer of +the server (a 404 for an unknown path) from the error page of a tunnel edge that could +not reach it (`relay/server.py`).""" IDEMPOTENCY_KEY_HEADER = "Idempotency-Key" IDEMPOTENCY_CACHE_TTL_SECONDS = 600 IDEMPOTENCY_CACHE_MAX_COMPLETED = 64 @@ -136,6 +141,10 @@ async def _request_digest(raw: bytes) -> bytes: return await asyncio.to_thread(_body_digest, raw) +async def _mark_response(request: web.Request, response: web.StreamResponse) -> None: + response.headers[SERVER_HEADER] = "1" + + def _error_response(dialect: Dialect, message: str, *, status: int) -> web.Response: """Keep the native error classification consistent with its HTTP status.""" return web.json_response( @@ -370,6 +379,9 @@ def __init__( make_tunnel(self.config.tunnel) if requires_tunnel else None ) self.healthy = True + self._replacing: asyncio.Task | None = None + self._lost_again = False + """A tunnel was lost while a replacement was under way: replace it once more.""" if self.tunnel is not None: self.tunnel.on_gone = self._handle_tunnel_gone self.host = "127.0.0.1" @@ -383,12 +395,82 @@ def load(self) -> int: return len(self.sessions) def _handle_tunnel_gone(self) -> None: - """Fail this server's rollouts and take no more: their URL will never work again.""" + """Take no more rollouts: the URL will never work again. Rollouts that can move + (`RolloutSession.reroute`) follow this server to a new tunnel; the others fail.""" self.healthy = False logger.error("Lost the interception tunnel url=%s", self.base_url) + movable = False for session in list(self.sessions.values()): - session.error = TunnelError("interception tunnel lost") - session.release() + if session.reroute is None: + session.error = TunnelError("interception tunnel lost") + session.release() + else: + movable = True + if not movable: + return + if self._replacing is None or self._replacing.done(): + self._replacing = asyncio.create_task(self._replace_tunnels()) + else: + self._lost_again = True + + async def _replace_tunnels(self) -> None: + """Replace the tunnel until no loss arrived while a replacement was under way.""" + while True: + self._lost_again = False + await self._replace_tunnel() + if not self._lost_again: + return + + async def _replace_tunnel(self) -> None: + """Open a new tunnel to this server and reroute its movable rollouts to it. A + rollout that does not move fails as on a lost tunnel.""" + assert self.tunnel is not None + try: + base_url = await self.stack.enter_async_context( + self.tunnel.expose(self.port) + ) + except TunnelError as error: + logger.error("interception tunnel replacement failed: %s", error) + base_url = None + else: + logger.warning( + "interception tunnel replaced: url=%s -> %s", self.base_url, base_url + ) + self.base_url = base_url + + async def move(session: RolloutSession) -> None: + try: + if base_url is None: + raise TunnelError("interception tunnel lost") + assert session.reroute is not None + await session.reroute(base_url) + except Exception as error: # noqa: BLE001 - one rollout's failure is its own + logger.warning( + "rollout %s did not move to the new tunnel: %s", + session.trace.id, + error, + ) + session.error = ( + error + if isinstance(error, RolloutError) + else TunnelError(f"interception tunnel lost: {error}") + ) + session.release() + + await asyncio.gather( + *( + move(session) + for session in list(self.sessions.values()) + if session.reroute is not None and not session.released + ) + ) + + async def stop(self) -> None: + if self._replacing is not None: + self._replacing.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._replacing + await super().stop() def _client(self, config: BaseClientConfig) -> Client: """The server-owned client for `config` — one per distinct endpoint config, shared @@ -574,6 +656,7 @@ async def start(self) -> None: app = web.Application( client_max_size=MAX_REQUEST_BODY, middlewares=[_isolate_log_scope] ) + app.on_response_prepare.append(_mark_response) app.router.add_get(HEALTH_PATH, self.handle_health) for dialect in DIALECTS: for route in dialect.routes: @@ -619,16 +702,30 @@ async def start(self) -> None: ) def _fail( - self, session: RolloutSession, dialect: Dialect, error: RolloutError + self, + session: RolloutSession, + dialect: Dialect, + error: RolloutError, ) -> web.Response: - """Stash a model-turn-adjacent failure (such as a hook raising) so the rollout - re-raises it as the real cause, and report it to the harness as an HTTP error.""" + """Stash a model-turn-adjacent failure so the rollout re-raises it as the real + cause, and report it to the harness as an HTTP error.""" session.error = error logger.warning("Failed turn - %s: %s", type(error).__name__, clip(str(error))) return _error_response( dialect, str(error), status=getattr(error, "status_code", 502) ) + async def _abort(self, session: RolloutSession, error: RolloutError) -> NoReturn: + """End the rollout's segment with a request hook's failure. The harness gets no + reply: the rollout cancels its turn, and releasing the session cancels this + handler.""" + logger.warning( + "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error + ) + session.abort(error) + await asyncio.get_running_loop().create_future() + raise AssertionError("unreachable") + def mediate_capabilities( self, session: RolloutSession, dialect: Dialect, body: dict ) -> tuple[dict, list[str]]: @@ -659,6 +756,7 @@ async def handle_tool(self, request: web.Request) -> web.Response: if session.released: return web.json_response({"error": "rollout concluded"}, status=409) body = from_json(await request.read()) + await session.flowing.wait() try: return web.json_response( await session.decide_tool( @@ -815,6 +913,7 @@ async def handle_request( if idempotent is not None and idempotent.response is not None: logger.debug("Replayed an idempotent request") idempotent.completed_at = now + await session.flowing.wait() return _replay_response(idempotent.response) try: @@ -850,6 +949,7 @@ async def coalesced( response = await asyncio.shield(inflight) if response is None: return _error_response(dialect, "upstream attempt failed", status=503) + await session.flowing.wait() return _replay_response(response) if idempotent.inflight is not None: @@ -884,11 +984,10 @@ async def coalesced( if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) except RolloutError as error: - return self._fail(session, dialect, error) + return await self._abort(session, error) except Exception as error: # noqa: BLE001 - surface task hook failures - return self._fail( + return await self._abort( session, - dialect, TaskError( f"model boundary hook failed: {type(error).__name__}: {error}" ), @@ -946,6 +1045,15 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + # The box is idle while the harness awaits this turn, so a snapshot captured + # before the reply returns matches the prompt exactly (tool results in, this + # reply not yet). It runs alongside the model call; the reply waits for it. + capture = ( + asyncio.create_task(session.checkpointer()) + if session.checkpointer is not None + else None + ) + try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1037,24 +1145,44 @@ async def sample() -> web.Response: error = e raise finally: - if node is None: - turn.abandon() - # The turn's one per-exchange record: settings, timing, outcome, and - # the error that ended it (if any). - self.record_call( - session, - dialect, - body, - started, - node=node, - finish_reason=call_response.finish_reason - if call_response - else None, - usage=call_response.usage if call_response else None, - error=error, - policy_paths=policy_paths, - acp=acp, - ) + try: + if capture is not None: + held = time.monotonic() + try: + # Shielded: a cancelled exchange leaves the snapshot to + # finish in the background, where the rollout awaits it. + checkpoint = await asyncio.shield(capture) + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", + session.trace.id, + exc_info=True, + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held + finally: + if node is None: + turn.abandon() + # The turn's one per-exchange record: settings, timing, outcome, + # and the error that ended it (if any). + self.record_call( + session, + dialect, + body, + started, + node=node, + finish_reason=call_response.finish_reason + if call_response + else None, + usage=call_response.usage if call_response else None, + error=error, + policy_paths=policy_paths, + acp=acp, + ) + # A memory snapshot under way must not see this reply reach the box. + await session.flowing.wait() return serve(call_response, events) if streaming: @@ -1191,6 +1319,7 @@ async def handle_state_put(self, request: web.Request) -> web.Response: {"error": f"invalid state PUT for {state_cls.__name__}: {e}"}, status=400, ) + await session.flowing.wait() if session.released: # the trace is sealed — a straggler write must not land return web.json_response({"error": "rollout concluded"}, status=409) session.trace.state = new_state diff --git a/verifiers/v1/restore.py b/verifiers/v1/restore.py new file mode 100644 index 0000000000..f54bb4130e --- /dev/null +++ b/verifiers/v1/restore.py @@ -0,0 +1,164 @@ +"""Saving running rollouts, so another process can continue them. + +A retry restores a checkpoint in the process that took it, from the live `Checkpoint`. +An owner that outlives its process (a training run that stops and resumes) saves every +running rollout instead (`save_all`): each one takes a memory checkpoint now and +returns its trace as plain data (`restorable`), which holds everything a restore +needs. A later process hands that trace back (`RunSlot.restore`, `Agent.run(restore=)`), +and the rollout continues in a new box from the checkpoint: its trace up to there, and +the harness program the box still runs. + +The owner holds the snapshot of a saved rollout (`Checkpoint.pinned`): no rollout +deletes it, and the owner deletes it once nothing will restore it +(`RuntimeBackend.discard`). +""" + +from __future__ import annotations + +import asyncio +import weakref +from contextvars import ContextVar +from typing import TYPE_CHECKING, Any + +from pydantic import TypeAdapter + +from verifiers.v1.session import ReplayResponse +from verifiers.v1.state import state_cls +from verifiers.v1.trace import Checkpoint, Trace +from verifiers.v1.types import ToolMessage +from verifiers.v1.utils.logging import get_logger + +if TYPE_CHECKING: + from verifiers.v1.configs.agent import AgentConfig + from verifiers.v1.harness import Harness + from verifiers.v1.rollout import Rollout + from verifiers.v1.task import Task + +logger = get_logger(__name__) + +LIVE: weakref.WeakSet[Rollout] = weakref.WeakSet() +"""The rollouts of this process that run a box (between `open` and `close`).""" + +restoring: ContextVar[list[dict[str, Any]] | None] = ContextVar( + "vf_restoring", default=None +) +"""The saved trace the episode this task runs continues from (`Env.run_episode`), in a +list its first agent run takes it out of.""" + + +def restorable(trace: Trace, checkpoint: Checkpoint) -> dict[str, Any]: + """`trace` as plain data that any process can continue from `checkpoint`: its dump + with token arrays, and on the checkpoint's entry what the checkpoint holds only + while its rollout lives (`Checkpoint.restore`). Every value survives a pickle or a + msgpack round trip with raw bytes and numpy arrays.""" + data = trace.model_dump(context={"token_arrays": True}) + entry = next(c for c in data["checkpoints"] if c["id"] == checkpoint.id) + entry["restore"] = { + "state": TypeAdapter(type(checkpoint.state)).dump_python( + checkpoint.state, mode="json" + ), + "info": dict(checkpoint.info or {}), + "pins": [ + (prefix, call_id, message.model_dump(mode="json")) + for (prefix, call_id), message in checkpoint.pins.items() + ], + "replays": { + key: (path, digest, reply.status, reply.body, reply.content_type) + for key, ((path, digest), reply) in checkpoint.replays.items() + }, + "live": checkpoint.live, + } + return data + + +def restore_entry(trace: dict[str, Any]) -> dict[str, Any]: + """The checkpoint entry a saved trace continues from: the newest one that holds + what a restore needs.""" + for entry in reversed(trace.get("checkpoints") or []): + if entry.get("restore") is not None: + return entry + raise ValueError(f"trace {trace.get('id')} holds no checkpoint to continue from") + + +def open_restore(data: dict[str, Any], task: Task) -> tuple[Trace, Checkpoint]: + """The trace and its live checkpoint, for `task`'s rollout to restore.""" + saved = restore_entry(data) + trace = Trace.model_validate(data) + checkpoint = next(c for c in trace.checkpoints if c.id == saved["id"]) + held, checkpoint.restore = saved["restore"], None + trace.state = state_cls(type(task)).model_validate(held["state"]) + checkpoint.state = trace.state.model_copy(deep=True) + checkpoint.info = dict(held["info"]) + checkpoint.pins = { + (prefix, call_id): ToolMessage.model_validate(message) + for prefix, call_id, message in held["pins"] + } + checkpoint.replays = { + key: ((path, digest), ReplayResponse(status, body, content_type)) + for key, (path, digest, status, body, content_type) in held["replays"].items() + } + checkpoint.live = held["live"] + return trace, checkpoint + + +def unsaveable(config: AgentConfig, harness: Harness | None = None) -> str | None: + """Why the rollouts of an agent with `config` take no memory snapshots, so `save_all` + returns nothing for them; None when they take them. A task with tool + servers or services of its own rules its rollout out as well, which only the + rollout knows.""" + from verifiers.v1.runtimes import backend_cls + from verifiers.v1.utils.loaders import harness_class + + if config.checkpoint is None: + return "rollout checkpointing is off (agent.checkpoint)" + if not backend_cls(config.runtime).SUPPORTS_MEMORY_SNAPSHOTS: + return f"the {config.runtime.type!r} runtime takes no memory snapshots" + if harness is not None: + supported, name = harness.SUPPORTS_MEMORY_RESTORE, harness.config.id + elif config.harness is not None: + supported = harness_class(config.harness.id).SUPPORTS_MEMORY_RESTORE + name = config.harness.id + else: + return None + if not supported: + return f"harness {name!r} cannot continue from a memory snapshot" + return None + + +def release_all() -> None: + """Let every rollout that `save_all(hold=True)` holds go on.""" + for rollout in list(LIVE): + rollout.release() + + +async def save_all( + skipped: dict[str, int] | None = None, *, hold: bool = False +) -> list[dict[str, Any]]: + """Take a memory checkpoint of every running rollout of this process and return + their traces as saved (`restorable`). A rollout that takes no memory snapshots, + runs no harness segment right now, or whose snapshot fails has none: it starts + over when its owner runs it again. `skipped` collects why, as a count per reason. + The snapshots are pinned: the caller owns them. With `hold`, each saved rollout + waits at its checkpoint until `release_all`, so the caller can record which + rollouts are still running without one of them finishing meanwhile.""" + + async def save(rollout: Rollout) -> dict[str, Any] | None: + try: + point = await rollout.save(hold=hold) + reason = rollout.save_skipped + except Exception: + logger.warning("saving rollout %s failed", rollout.trace.id, exc_info=True) + point, reason = None, "its save raised an error" + if point is None and skipped is not None: + reason = reason or "unknown" + skipped[reason] = skipped.get(reason, 0) + 1 + return point + + rollouts = list(LIVE) + points = [ + point + for point in await asyncio.gather(*(save(rollout) for rollout in rollouts)) + if point is not None + ] + logger.info("Saved %d of %d running rollouts", len(points), len(rollouts)) + return points diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 9d7da6a56f..84cbf25da5 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -2,19 +2,24 @@ import asyncio import contextlib +import copy import logging import time +import uuid from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from types import EllipsisType +from typing import Any, Literal from urllib.parse import urlsplit from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.agent import AgentConfig +from verifiers.v1.configs.agent import AgentConfig, RolloutCheckpointConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.configs.scoring import resolve_scoring from verifiers.v1.errors import ( HarnessError, + RelayError, RolloutError, SandboxError, TaskError, @@ -30,18 +35,28 @@ ) from verifiers.v1.harness import Harness, HarnessSession from verifiers.v1.interception import Interception, serve_interception +from verifiers.v1.interception.relay import Relay from verifiers.v1.mcp import SharedToolServer, serve_tools +from verifiers.v1.restore import LIVE, restorable, unsaveable from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, + backend_cls, make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms -from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary +from verifiers.v1.session import ( + IdempotentRequest, + RolloutLimits, + RolloutSession, + hook_boundary, +) from verifiers.v1.state import state_cls from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage +from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke from verifiers.v1.utils.format import format_time @@ -72,6 +87,12 @@ class RolloutTimeouts: isolated verifier setup and retries. Finalization has its own budget.""" +HOLD_SECONDS = 120 +"""How long a saved rollout waits for its owner's `release` before it goes on by itself.""" +REPLAY_SECONDS = 30 +"""A memory checkpoint keeps the replies the server sent this recently: one of them may +not have reached the box when the snapshot stopped it.""" + _HOME_PROBE = ( "home=$(awk -F: -v u=\"$1\" '$1 == u { print $6 }' /etc/passwd); " 'test -n "$home" || exit 1; home=${2:-$home}; echo "$home"; test -d "$home"' @@ -106,6 +127,23 @@ async def resolve_agent_runtime( return view +def kept(checkpoints: list[Checkpoint], keep: Literal["last"] | int) -> set[str]: + """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every + k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + live = [ + (index, c) + for index, c in enumerate(checkpoints) + if c.time.end and not c.deleted and not c.pinned + ] + if not live: + return set() + held = {live[-1][1].id} + if keep != "last": + held.update(c.id for index, c in live if index % keep == 0) + return held + + class Rollout: """Manages one rollout's lifecycle (open, step, close).""" @@ -128,7 +166,17 @@ def __init__( on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, trace_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, ) -> None: + """`attempt` numbers this run among the trace's retries, from 1 (stamped on + the trace and the errors it captures). `restore` continues a failed attempt's trace from one of + its checkpoints: the trace rewinds to it and this run boots its box from the + snapshot instead of setting the task up again. `checkpoint` snapshots the box as + model turns go upstream (on a runtime and a harness that support it) for a + retry to restore, and deletes the ones its `keep` policy drops as newer ones + become restorable; the owner of the retries deletes the rest once the run is + final — this rollout deletes them only when it aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -148,23 +196,47 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - **({"id": trace_id} if trace_id is not None else {}), - attempt=attempt, - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) + self._checkpointing = checkpoint + self._durable: dict[str, asyncio.Task[None]] = {} + self._captures: set[asyncio.Task] = set() + """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" + """The snapshots still on their way to restorable, by checkpoint id.""" + self._attempt = attempt + self._restore: Checkpoint | None = None + self.restored_turns: int | None = None + """The model turns the restored trace held when this run began (None for a run + that did not restore). The next turn is the one the snapshot was taken at, which + this run does not snapshot again.""" + self.deleted_snapshots: set[str] = set() + """The snapshots this run deleted.""" + self._pruning = asyncio.Lock() + self.rewound_snapshots: list[Snapshot] = [] + """The snapshots of the checkpoints the restored trace dropped (taken after the + one restored).""" + if restore is not None: + self.trace, self._restore = restore + self.trace.attempt = attempt + self.rewound_snapshots = self.trace.rewind(self._restore) + self.restored_turns = self.trace.num_turns + self._restore.restored = True + else: + self.trace = Trace( + **({"id": trace_id} if trace_id is not None else {}), + attempt=attempt, + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) self.trace._replay_view = harness.replay_view - if on_trace is not None: - on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -194,6 +266,28 @@ def __init__( response_stops=[fn for boundary, fn in stops if boundary is Response], gates_tools=harness.SUPPORTS_TOOL_INTERCEPTION, ) + self._memory = False + """Whether this run can take memory snapshots: decided at `open`, once the box, + its relay and the task's tool servers are known. Its checkpoints at model turns + take them when the config asks for them, and its owner's saves always do.""" + self.unsaveable: str | None = "it has not booted its box yet" + """Why this run takes no memory snapshots, so `save` returns nothing; None when + it takes them.""" + self.save_skipped: str | None = None + """Why the last `save` returned nothing.""" + self._held = False + """Whether an owner holds the box's replies past its save (`save(hold=True)`).""" + self._release_timer: asyncio.TimerHandle | None = None + self._capturing = asyncio.Lock() + if self._restore is not None: + # Set before the box boots: its relay repeats the requests that waited as + # soon as it reaches this session. + self._session.pinned_tool_results.update(self._restore.pins) + now = time.monotonic() + for key, (binding, response) in self._restore.replays.items(): + self._session.idempotent_requests[key] = IdempotentRequest( + binding=binding, response=response, completed_at=now + ) self._stack = AsyncExitStack() self._failed = False self._failure: Exception | None = None @@ -202,6 +296,8 @@ def __init__( self._closed = False self._harness_cleaned = False self._endpoint: str | None = None + self._relay: Relay | None = None + """The relay a harness in a remote box reaches the interception server through.""" self._urls: dict[str, str] = {} self._harness_session: HarnessSession | None = None self._agent_runtime: Runtime | None = None @@ -226,10 +322,11 @@ def failure(self) -> Exception | None: """The original exception most recently captured onto the trace.""" return self._failure - def fail(self, error: Exception) -> None: + def fail(self, error: Exception, stage: str | None = None) -> None: """Record `error` as this rollout's outcome (captured onto the trace, the remaining stages skipped) — the run's owner reporting a failure the run - itself couldn't see, e.g. its user raising between segments.""" + itself couldn't see, e.g. its user raising between segments. `stage` names + the rollout stage it broke in, when known.""" if ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -246,7 +343,7 @@ def fail(self, error: Exception) -> None: logger.exception("Failed rollout with an unexpected error") self._failed = True self._failure = error - self.trace.record_error(error) + self.trace.record_error(error, attempt=self._attempt, stage=stage) def timeout(self, stage: str) -> None: """Record `stage`'s expired deadline as this rollout's outcome: a stop, not @@ -260,10 +357,20 @@ async def open(self) -> bool: interception slot and tool servers. Returns whether the exchange can proceed; a setup failure is captured onto the trace.""" self._opened = True + LIVE.add(self) self.trace.timing.boot.start = time.time() self.trace.notify() if self._borrowed_runtime is None: - self.runtime = make_backend(self.runtime_config, name=self.trace.id).bind() + config = self.runtime_config + name = self.trace.id + if self._restore is not None: + # A restored trace keeps its id; its new box must not collide with the + # box an earlier attempt or process left behind. + name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) + self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -278,11 +385,21 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info.model_copy() - logger.debug( - "Starting rollout harness=%s runtime=%s", - self.harness.config.name, - self.runtime_config.type, - ) + if self._restore is None: + logger.debug( + "Starting rollout harness=%s runtime=%s", + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "Restoring rollout from checkpoint %s at turn %d harness=%s runtime=%s", + self._restore.id, + self.trace.num_turns, + self.harness.config.name, + self.runtime_config.type, + ) + restored = self._restore is not None loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -350,21 +467,21 @@ async def open(self) -> bool: self.trace.timing.boot.end = now self.trace.timing.setup.start = now self.trace.notify() - started = time.perf_counter() - async with ( - boundary(TaskError, "task setup"), - ): - await invoke( - self.task.setup, - { - "trace": self.trace, - "runtime": runtime, - "deployment": self.deployment, - }, + # A restored box already holds the task's setup. + if self._restore is None: + started = time.perf_counter() + async with boundary(TaskError, "task setup"): + await invoke( + self.task.setup, + { + "trace": self.trace, + "runtime": runtime, + "deployment": self.deployment, + }, + ) + logger.debug( + "Set up task in %s", format_time(time.perf_counter() - started) ) - logger.debug( - "Set up task in %s", format_time(time.perf_counter() - started) - ) started = time.perf_counter() async with ( boundary(HarnessError, "harness setup"), @@ -372,7 +489,41 @@ async def open(self) -> bool: self._agent_runtime = await resolve_agent_runtime( runtime, self.task.data.agent_user, self.task.data.agent_home ) - await self.harness.setup(self._agent_runtime) + # A harness in a remote box reaches the interception server through + # a relay in its box, which starts alongside the harness setup. + if not runtime.is_local: + self._relay = Relay( + runtime, + self.trace.id, + running=self._restore.live["relay"] if restored else None, + ) + self._stack.push_async_callback(self._relay.stop) + await self._relay.installed() + # A restored box still runs the harness. + if not restored: + await self.harness.setup(self._agent_runtime) + self.unsaveable = unsaveable(self.trace.agent.config, self.harness) or ( + "it runs in a borrowed box" + if self._borrowed_runtime is not None + # Everything the box reaches the host through must follow it to a + # new box: the relay does, tool servers and services do not. + else "its task has tool servers or services of its own" + if toolsets + or self._shared_tools + or self.deployment.spec is not None + else None + ) + self._memory = self.unsaveable is None and self._relay is not None + if restored and not self._memory: + raise SandboxError( + "the rollout cannot continue from its memory snapshot" + ) + if ( + self._checkpointing is not None + and self._borrowed_runtime is None + and self._memory + ): + self._session.checkpointer = self._checkpoint logger.debug( "Set up %s harness in %s", self.harness.config.name, @@ -383,6 +534,16 @@ async def open(self) -> bool: # The harness reaches the model at `{base_url}/v1`; tool servers reach this # rollout's `/state` + `/task` at `base_url` — it's universally reachable # (the interception is exposed whenever any consumer is remote). + # The relay follows the server to a replacement tunnel, unless tool + # servers or services reach the server too. Set before the server + # registers the session: a tunnel lost while the relay starts waits. + if ( + self._relay is not None + and not toolsets + and not self._shared_tools + and self.deployment.spec is None + ): + self._session.reroute = self._reroute ( base_url, model_secret, @@ -396,8 +557,20 @@ async def open(self) -> bool: self._shared_tools, ) ) - self._endpoint = runtime.host_url(f"{base_url.rstrip('/')}/v1") - self._secret = model_secret + harness_base = runtime.host_url(base_url.rstrip("/")).rstrip("/") + # A harness restored from memory still sends the secret it got at its + # launch; the relay swaps it for this session's. + self._secret = ( + self._restore.live["secret"] if restored else model_secret + ) + if self._relay is not None: + # The harness holds the model secret anyway; the relay gets + # nothing more, so reading its config grants no capability. + async with boundary(RelayError, "starting the relay"): + harness_base = await self._relay.connect( + base_url.rstrip("/"), {self._secret: model_secret} + ) + self._endpoint = f"{harness_base}/v1" services = ( { name: self.deployment.target(name) @@ -437,7 +610,11 @@ async def open(self) -> bool: await self.deployment.prepare_execution( [ url - for url in [self._endpoint, *self._urls.values()] + for url in [ + self._endpoint, + *self._urls.values(), + *([self._relay.upstream] if self._relay else []), + ] if urlsplit(url).hostname not in internal ], state_route=f"{base_url.rstrip('/')}/state", @@ -489,9 +666,11 @@ async def open(self) -> bool: harness_data = harness_data.model_copy( update={"system_prompt": network_prompt} ) + # A restored trace already carries its prompt's rewrites. if ( self._session.request_interceptors and harness_data.prompt is not None + and self._restore is None ): prompt = harness_data.prompt system_prompt = harness_data.system_prompt @@ -530,11 +709,7 @@ async def open(self) -> bool: ) if not self._session.stopped: session_kwargs = ( - { - "tool_interception_url": runtime.host_url( - f"{base_url.rstrip('/')}/tool" - ) - } + {"tool_interception_url": f"{harness_base}/tool"} if self.harness.SUPPORTS_TOOL_INTERCEPTION and ( self._session.request_interceptors @@ -552,6 +727,7 @@ async def open(self) -> bool: harness_data, **session_kwargs, ) + self._harness_session.memory = self._memory except TimeoutError as e: # Only our expired setup budget owns the timeout; a provider can # raise its own TimeoutError before the overall deadline. @@ -559,13 +735,14 @@ async def open(self) -> bool: self.timeout("setup") if not self._deployment_started: self.fail( - SandboxError("deployment provisioning exceeded setup timeout") + SandboxError("deployment provisioning exceeded setup timeout"), + stage="setup", ) else: - self.fail(e) + self.fail(e, stage="setup") return False except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure - self.fail(e) + self.fail(e, stage="setup") return False except BaseException: # A cancellation mid-setup kills the driver's await with it, so no @@ -580,6 +757,22 @@ async def open(self) -> bool: logger.debug("Starting agent") return not self._session.stopped + async def _reroute(self, base_url: str) -> None: + """Move the harness's relay to the interception server's new URL + (`RolloutSession.reroute`).""" + assert self._relay is not None + if self._closed: + # Closing: the agent is done and the relay unused, and scoring keeps its + # own egress policy. + return + await self._relay.wait_connected() + upstream = base_url.rstrip("/") + # A restricted box must reach the new tunnel host before the relay uses it. + await self.deployment.prepare_execution( + [self._endpoint, upstream], state_route=f"{upstream}/state" + ) + await self._relay.reroute(upstream) + async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed or not self.ok: raise RuntimeError("this rollout is closed or stopped") @@ -597,9 +790,9 @@ async def step( segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -628,15 +821,18 @@ async def step( return False self._session.on_turn_input = on_input assert self.deployment is not None - await self._harness_session.turn(messages) + if await self._turn(messages): + assert self._session.aborted is not None + self.fail(self._session.aborted, stage="agent") + return False except TimeoutError as e: # An expired rollout deadline is the agent breaking its time budget: a - # timeout stop. A TimeoutError from the harness's own I/O with no - # expired deadline stays the raw failure. + # timeout stop. A TimeoutError from the harness's own I/O with no expired + # deadline stays the raw failure. if self.deadline_at is not None and (loop.time() >= self.deadline_at): self.timeout("agent") else: - self.fail(e) + self.fail(e, stage="agent") return False except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure if self._session.stopped: @@ -648,11 +844,14 @@ async def step( self.timeout("agent") return False real = self._session.error + if real is None and self._relay is not None: + # A harness that dies on its relay's failure records the relay's. + real = await self._relay.failure() if real is not None and isinstance(e, RolloutError): real.__cause__ = e - self.fail(real) + self.fail(real, stage="agent") else: - self.fail(e) + self.fail(e, stage="agent") return False finally: self._session.on_turn_input = None @@ -672,6 +871,301 @@ async def step( # never moved, forever. return self.ok and trace.num_turns > turns_before + async def _turn(self, messages: Messages | None) -> bool: + """Run one harness segment. A request hook that fails ends it at once (the + session aborts): the harness's turn is cancelled, so the harness never handles + the failure. Returns whether the segment was aborted.""" + assert self._harness_session is not None + turn = asyncio.ensure_future(self._harness_session.turn(messages)) + aborted = asyncio.ensure_future(self._session.abort_event.wait()) + try: + await asyncio.wait({turn, aborted}, return_when=asyncio.FIRST_COMPLETED) + finally: + aborted.cancel() + if not turn.done(): + turn.cancel() + # Joined even through another cancellation: a turn left running + # could still write the trace or use the box during teardown. + await run_shielded(asyncio.wait({turn})) + if self._session.aborted is not None: + if not turn.cancelled(): + turn.exception() # the abort decides the outcome, whatever the turn did + return True + turn.result() + return False + + async def restore(self) -> bool: + """Run the restored attempt's first segment: the box restored from the + checkpoint still runs the harness program, so this waits for it + (`HarnessSession.reattach`) instead of starting one.""" + checkpoint = self._restore + assert checkpoint is not None and self._harness_session is not None + self._harness_session.reattach(checkpoint.live["program"]) + return await self.step() + + async def _checkpoint(self) -> Checkpoint | None: + if self._closed: + # Settling may have drained `_captures` already: a snapshot taken now + # would start its upload after the box is gone. + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + return await self._capture() + finally: + self._captures.discard(task) + + async def _capture(self) -> Checkpoint | None: + assert self._checkpointing is not None + interval = self._checkpointing.interval + turn_number = self.trace.num_turns + 1 + if ( + interval is None + or turn_number % interval + or ( + self.restored_turns is not None + and turn_number == self.restored_turns + 1 + ) + ): + return None + newest = next( + (c for c in reversed(self.trace.checkpoints) if not c.deleted), None + ) + if newest is not None and newest.node == self._last_node(): + # No turn since the newest snapshot, so it holds this same open call. A + # snapshot resets the box's connections and its relay asks again: a call + # that keeps failing would otherwise take a snapshot per request. + return None + return await self._take_snapshot(at=self._last_node()) + + def _last_node(self) -> int | None: + return len(self.trace.nodes) - 1 if self.trace.nodes else None + + async def save(self, *, hold: bool = False) -> dict[str, Any] | None: + """Take a memory checkpoint of the running rollout now and return its trace as + saved (`restorable`), for an owner that continues the rollout in another + process (`save_all`). The owner holds the snapshot (`Checkpoint.pinned`); it + becomes restorable from any node in the background, and this run's teardown + waits for that. With `hold`, nothing the box asked for is answered until + `release`, so the rollout cannot move past its checkpoint while the owner + records it. + None when the rollout takes no memory snapshots (`unsaveable`), runs no + segment, or the snapshot failed; `save_skipped` then says which.""" + self.save_skipped = None + if self._closed: + self.save_skipped = "it is finishing" + elif not self._memory: + self.save_skipped = self.unsaveable or "its box has no relay" + elif self._harness_session is None or ( + self._harness_session.memory_state() is None + ): + self.save_skipped = "its harness program is not running (setup or scoring)" + if self.save_skipped is not None: + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + # A save without a hold does not end the hold of an earlier one. + self._held = hold or self._held + checkpoint = None + try: + checkpoint = await self._take_snapshot(pinned=True) + if checkpoint is None: + self.release() + self.save_skipped = "its memory snapshot failed" + return None + saved = restorable(self.trace, checkpoint) + except BaseException: + # The owner never gets this save: nobody would release the hold or delete + # the snapshot, so both go back to the rollout. + if checkpoint is not None: + checkpoint.pinned = False + self.release() + raise + finally: + self._captures.discard(task) + if hold: + if self._release_timer is not None: + self._release_timer.cancel() + # An owner that never releases must not stall the rollout for good. + self._release_timer = asyncio.get_running_loop().call_later( + HOLD_SECONDS, self.release + ) + return saved + + def release(self) -> None: + """Answer the box again after a `save(hold=True)`.""" + self._held = False + if self._release_timer is not None: + self._release_timer.cancel() + self._release_timer = None + # A snapshot that is being taken holds the box for itself, and lets the + # replies flow when it is done. + if not self._capturing.locked(): + self._session.resume() + + async def _take_snapshot( + self, *, pinned: bool = False, at: int | None | EllipsisType = ... + ) -> Checkpoint | None: + """Snapshot the box with its memory. Nothing the box asked for is answered + while the snapshot is taken (`RolloutSession.hold`), so the box it stops has + seen exactly the trace as recorded here; a box restored from it asks again + for what was still open. With `at`, only while the trace still ends at that + node: a snapshot of a turn that waited for another one comes too late when + the turn's reply is on the trace and not yet in the box.""" + assert self.runtime is not None and self._relay is not None + session = self._session + if self._harness_session is None: + return None + async with self._capturing: + if self._closed or (at is not ... and at != self._last_node()): + return None + session.hold() + try: + program = self._harness_session.memory_state() + if program is None: + return None + node = self._last_node() + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(session.pinned_tool_results) + now = time.monotonic() + replays = { + key: (request.binding, request.response) + for key, request in session.idempotent_requests.items() + if request.response is not None + and request.completed_at is not None + and now - request.completed_at <= REPLAY_SECONDS + } + started = time.time() + try: + snapshot = await self.runtime.backend.snapshot() + except Exception: + logger.warning( + "memory snapshot failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + return None + captured = time.time() + # The box's connections to the server did not survive the stop: its + # relay opens new ones and repeats the requests that wait. + try: + await self._relay.reroute(self._relay.upstream) + except Exception: + logger.warning( + "relay did not reconnect after a memory snapshot (rollout %s)", + self.trace.id, + exc_info=True, + ) + finally: + if not self._held: + session.resume() + checkpoint = Checkpoint( + id=snapshot.id, + node=node, + state=state, + info=info, + pins=pins, + replays=replays, + live={ + "program": program, + "relay": self._relay.state, + "secret": self._secret, + }, + snapshot=snapshot, + time=TimeSpan(start=started), + captured=captured, + pinned=pinned, + ) + self.trace.checkpoints.append(checkpoint) + self._durable[checkpoint.id] = asyncio.create_task( + self._make_durable(checkpoint) + ) + logger.debug( + "memory checkpoint %s taken in %s: id=%s node=%s", + checkpoint.id, + format_time(captured - started), + self.trace.id, + node, + ) + return checkpoint + + async def _make_durable(self, checkpoint: Checkpoint) -> None: + assert self.runtime is not None + try: + checkpoint.bytes = await self.runtime.backend.wait_snapshot( + checkpoint.snapshot + ) + except Exception: + # Not restorable: no retry may pick it. The backend still holds it as + # stray and frees it with the box. + logger.warning( + "checkpoint %s never became durable (rollout %s)", + checkpoint.id, + self.trace.id, + exc_info=True, + ) + with contextlib.suppress(ValueError): + self.trace.checkpoints.remove(checkpoint) + else: + checkpoint.time.end = time.time() + self.trace.notify() + await self._prune() + finally: + self._durable.pop(checkpoint.id, None) + + async def _prune(self) -> None: + """Delete the snapshots the `keep` policy drops now that a newer one is + restorable, and the ones a restore rewound past.""" + assert self.runtime is not None and self._checkpointing is not None + async with self._pruning: + keep = kept(self.trace.checkpoints, self._checkpointing.keep) + drop = [ + *( + c.snapshot + for c in self.trace.checkpoints + if c.time.end + and not c.deleted + and not c.pinned + and c.id not in keep + ), + *( + s + for s in self.rewound_snapshots + if s.id not in self.deleted_snapshots + ), + ] + if drop: + self.record_deleted(await self.runtime.backend.discard_snapshots(drop)) + + def record_deleted(self, ids: list[str]) -> None: + """Mark the snapshots with these ids deleted.""" + self.deleted_snapshots.update(ids) + for checkpoint in self.trace.checkpoints: + if checkpoint.id in self.deleted_snapshots: + checkpoint.deleted = True + + async def _settle_checkpoints(self, *, wait: bool) -> None: + """Before the box goes: wait for the snapshots still uploading, so a retry + can restore them (its teardown would fail them), or, with nothing to retry, + stop waiting and leave them to be freed.""" + # A snapshot still being taken would start its upload after this returned. + await asyncio.gather(*self._captures, return_exceptions=True) + tasks = list(self._durable.values()) + if not wait: + # An owner waits for its snapshots whatever becomes of this run (`save`). + pinned = {c.id for c in self.trace.checkpoints if c.pinned} + for checkpoint_id, task in self._durable.items(): + if checkpoint_id not in pinned: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + # A task cancelled before it first ran never reached its own cleanup. + for checkpoint_id, task in list(self._durable.items()): + if task.done(): + del self._durable[checkpoint_id] + async def _cleanup_harness(self, runtime: Runtime) -> None: if self._harness_cleaned: return @@ -692,6 +1186,8 @@ async def abort(self) -> None: (a cancellation mid-setup, a lifetime bug raised to the caller) means the driver will never reach `close()`. Safe after a partial `close()`.""" self._closed = True + LIVE.discard(self) + await self._settle_checkpoints(wait=False) if self._harness_session is not None: with contextlib.suppress(Exception): await self._harness_session.close() @@ -710,6 +1206,22 @@ async def abort(self) -> None: ): with contextlib.suppress(Exception): await self.runtime.backend.stop() + if self.runtime is not None: + # Strays were never restorable, so nobody else knows of them. + with contextlib.suppress(Exception): + self.record_deleted( + await self.runtime.backend.discard_snapshots( + [ + *self.rewound_snapshots, + *( + c.snapshot + for c in self.trace.checkpoints + if not c.deleted and not c.pinned + ), + *self.runtime.backend.stray_snapshots, + ] + ) + ) async def _score_harness(self, runtime: Runtime, deadline: float | None) -> None: """Collect best-effort metrics without replacing a task failure.""" @@ -802,6 +1314,7 @@ async def close(self) -> Trace: if self._closed: return self.trace self._closed = True + LIVE.discard(self) trace = self.trace skip_reason = self._scoring_skip_reason() runtime = self.runtime @@ -983,7 +1496,7 @@ async def close(self) -> Trace: "Scored rollout in %s", format_time(trace.timing.scoring.duration) ) except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure - self.fail(e) + self.fail(e, stage="scoring" if trace.timing.scoring.start else "finalize") finally: if self._harness_session is not None: with contextlib.suppress(Exception): @@ -1005,6 +1518,9 @@ async def close(self) -> Trace: trace.split_agent_time() if runtime is not None: await self._cleanup_harness(runtime) + # A failed attempt's retry restores the newest snapshot, which must be + # durable before the box goes. + await self._settle_checkpoints(wait=not trace.ok) # Tear down here — the env's `score()` (later) needs only the traces, # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. @@ -1019,6 +1535,10 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() + if runtime is not None and runtime.backend.stray_snapshots: + await runtime.backend.discard_snapshots( + runtime.backend.stray_snapshots + ) except Exception as e: logger.warning( "Failed to tear down runtime - %s: %s", diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 46f82c44f6..8538055967 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -25,9 +25,11 @@ ) from verifiers.v1.runtimes.base import ( BaseRuntimeInfo, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -102,12 +104,14 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "PrimeBackend", "PrimeConfig", "PrimeRuntimeInfo", + "Program", "ProgramResult", "Runtime", "RuntimeBackend", "RuntimeConfig", "RuntimeInfo", "RuntimeProcess", + "Snapshot", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index f07fae54f4..3030abdc2c 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -24,9 +24,11 @@ from verifiers.v1.runtimes.base import ( SERVICE_PORT, BaseRuntimeInfo, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, parse_gpu, ) from verifiers.v1.runtimes.limiters import creation_limiter @@ -43,6 +45,16 @@ EFFECTIVELY_UNBOUNDED_SECONDS = 30 * 24 * 60 * 60 """Safety deadline for APIs that require a finite bound. Normal execution remains bounded by idle detection or rollout cancellation; 30 days is above any real run.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +_PAUSE_SETTLE_SECONDS = 120 +"""How long a pause or a resume gets to take effect; they take seconds.""" +_DEAD_STATUSES = frozenset({"ERROR", "TERMINATED", "TIMEOUT"}) +_STATUS_POLL_SECONDS = 0.3 +_FORK_ATTEMPTS = 3 +"""Creates of a box from a memory snapshot before its restore counts as failed.""" +_RESUME_RETRY_SECONDS = 600 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it +after the request returns, and only a durable one restores.""" _OUTPUT_TAIL_BYTES = 10 * 1024 * 1024 """Maximum retained bytes per command output stream.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" @@ -54,10 +66,62 @@ BASE_LABELS: list[str] = [] +class _StatusPoller: + """Batched status lookups for the boxes that wait for a pause or a resume to take + effect: one request serves up to 100 boxes, so a save of every rollout of a run + costs a few requests per poll instead of one per box.""" + + def __init__(self, client: Any) -> None: + self.client = client + self.waiting: dict[str, asyncio.Future] = {} + self.task: asyncio.Task | None = None + + async def next(self, sandbox_id: str) -> Any: + """The box's status at the next poll (one waiter per box).""" + future = asyncio.get_running_loop().create_future() + self.waiting[sandbox_id] = future + if self.task is None or self.task.done(): + self.task = asyncio.create_task(self._run()) + return await future + + async def _run(self) -> None: + while self.waiting: + await asyncio.sleep(_STATUS_POLL_SECONDS) + waiting, self.waiting = self.waiting, {} + ids = list(waiting) + await asyncio.gather( + *( + self._poll({i: waiting[i] for i in ids[start : start + 100]}) + for start in range(0, len(ids), 100) + ) + ) + + async def _poll(self, waiting: dict[str, asyncio.Future]) -> None: + try: + response = await self.client.get_sandbox_statuses(list(waiting)) + except Exception as error: # noqa: BLE001 - each waiter reports it + results: dict[str, Any] = dict.fromkeys(waiting, error) + else: + results = {status.sandbox_id: status for status in response.statuses} + results.update( + (item.sandbox_id, SandboxError(f"{item.code}: {item.message}")) + for item in response.errors + ) + for sandbox_id, future in waiting.items(): + result = results.get(sandbox_id, SandboxError("no status returned")) + if future.done(): + continue + if isinstance(result, Exception): + future.set_exception(result) + else: + future.set_result(result) + + @dataclass class _SharedClient: client: Any leases: int = 0 + poller: _StatusPoller | None = None # One `AsyncSandboxClient` per event loop, leased by every live runtime on it. The SDK @@ -142,6 +206,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True + SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -159,6 +224,12 @@ def __init__(self, config: PrimeConfig, name: str | None = None) -> None: self._expose_network_id: str | None = None self._exposed_ports: list[int] = [] self._expose_lock = asyncio.Lock() + self._awake = asyncio.Event() + """Cleared while a memory snapshot stops the box: the platform refuses every + call into a stopped box, so calls wait for it.""" + self._awake.set() + self._snapshots = 0 + """Memory snapshots taken of this box. Each one ended the open streams into it.""" @property def published_port(self) -> int: @@ -170,7 +241,11 @@ def bind(self, *, env: Mapping[str, str] | None = None) -> Runtime: return super().bind(env=env).with_user(self.user) async def start(self) -> None: - from prime_sandboxes import APIError, CreateSandboxRequest + from prime_sandboxes import ( + APIError, + CreateSandboxRequest, + SandboxNotRunningError, + ) started = time.perf_counter() self._client = _lease_client() @@ -198,7 +273,13 @@ async def start(self) -> None: {"mount_path": target, **mount.model_dump()} for target, mount in self.config.mounts.items() ] - if self.config.checkpoint: + memory = bool(self.config.checkpoint and self.config.restore_memory) + if memory: + # The box continues the snapshot's own processes: the platform fixes its + # shape, environment and volumes to the paused box's and refuses others. + source = {"checkpoint_id": self.config.checkpoint, "restore_memory": True} + options = {"timeout_minutes": -1, "idle_timeout_minutes": idle_minutes} + elif self.config.checkpoint: source = {"checkpoint_id": self.config.checkpoint} else: source = { @@ -210,60 +291,86 @@ async def start(self) -> None: scope = run_scope() labels = [*BASE_LABELS, *self.config.labels, scope] try: - async with ( - creation_limiter( - (self.config.creates_per_min or 0) / 60, - "prime-sandbox", - scope, - ) - or contextlib.nullcontext() - ): - for name in dict.fromkeys(m.name for m in self.config.mounts.values()): - try: - await self._client.volumes.create(name) - except APIError as error: - # Create returns 409 when this scoped name already exists, - # including when another runtime creates it concurrently. - if not str(error).startswith("HTTP 409:"): - raise - - # Shielded through the id capture: a cancel that aborts the POST - # mid-flight leaves the platform creating a sandbox this side - # never learned the id of — teardown() then cannot delete it - async def create_and_capture_id(): - sandbox = await self._client.create( - CreateSandboxRequest( - name=self.name, - labels=list(dict.fromkeys(labels)), - **source, - environment_vars=self.env, - **{k: v for k, v in options.items() if v is not None}, + attempts = _FORK_ATTEMPTS if memory else 1 + for attempt in range(attempts): + async with ( + creation_limiter( + (self.config.creates_per_min or 0) / 60, + "prime-sandbox", + scope, + ) + or contextlib.nullcontext() + ): + for name in dict.fromkeys( + m.name for m in self.config.mounts.values() + ): + try: + await self._client.volumes.create(name) + except APIError as error: + # Create returns 409 when this scoped name already exists, + # including when another runtime creates it concurrently. + if not str(error).startswith("HTTP 409:"): + raise + + # Shielded through the id capture: a cancel that aborts the POST + # mid-flight leaves the platform creating a sandbox this side + # never learned the id of — teardown() then cannot delete it + async def create_and_capture_id(): + sandbox = await self._client.create( + CreateSandboxRequest( + name=self.name, + labels=list(dict.fromkeys(labels)), + **({} if memory else {"environment_vars": self.env}), + **source, + **{k: v for k, v in options.items() if v is not None}, + ) ) + self.info.id = sandbox.id + return sandbox + + sandbox = await run_shielded(create_and_capture_id()) + # The create response says whether the platform already has the image: + # `pending_image_build_id` set means a first-use auto-build is running and the + # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase + # its own budget, separate from the normal boot attempts). + self.info.image_cached = sandbox.pending_image_build_id is None + if not self.info.image_cached: + logger.warning( + "Building image %s on the platform for prime runtime %s - the " + "first use of an image can take ~10 minutes, later runs start " + "in seconds", + self.config.image, + self.info.id, + ) + try: + await self._client.wait_for_creation(self.info.id, max_attempts=900) + break + except SandboxNotRunningError as error: + # A box that continues an old memory snapshot also continues its + # idle clock: the platform can end it as idle before it runs. + if attempt + 1 == attempts: + raise + logger.warning( + "Prime runtime %s ended before it ran from memory snapshot " + "%s; creating it again (attempt %d/%d) - %s", + self.info.id, + self.config.checkpoint, + attempt + 2, + attempts, + error, ) - self.info.id = sandbox.id - return sandbox - - sandbox = await run_shielded(create_and_capture_id()) - # The create response says whether the platform already has the image: - # `pending_image_build_id` set means a first-use auto-build is running and the - # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase - # its own budget, separate from the normal boot attempts). - self.info.image_cached = sandbox.pending_image_build_id is None - if not self.info.image_cached: - logger.warning( - "Building image %s on the platform for prime runtime %s - the " - "first use of an image can take ~10 minutes, later runs start " - "in seconds", - self.config.image, - self.info.id, - ) - await self._client.wait_for_creation(self.info.id, max_attempts=900) logger.debug( - "Started prime runtime %s in %s image=%s", + "Started prime runtime %s in %s image=%s%s", self.info.id, format_time(time.perf_counter() - started), self.config.image, + f" checkpoint={self.config.checkpoint}" + if self.config.checkpoint + else "", ) + if memory: + # The paused box was already prepared; its processes run again. + return await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" ) @@ -279,6 +386,154 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def snapshot(self) -> Snapshot: + # A pause is the platform's memory snapshot: it stops the box, and the resume + # continues it. The platform uploads the snapshot after the pause. Until it is + # durable the snapshot is stray: the platform refuses to delete it while this + # box lives, so teardown frees it after. + async def create() -> Snapshot: + # Shielded as one: a box left paused never runs its rollout again. + self._awake.clear() + self._snapshots += 1 + try: + paused = await self._client.pause(self.info.id) + snapshot = Snapshot(id=paused.checkpoint_id) + # Kept whatever comes next: a pause that was not seen to finish can + # still have made the snapshot. + self.stray_snapshots.append(snapshot) + try: + await self._settle("PAUSED", "PAUSE_FAILED") + finally: + # The pause was accepted, so the box stops or has stopped. It must + # run again also when the wait for it failed. + await self._resume() + finally: + self._awake.set() + return snapshot + + try: + async with ( + creation_limiter( + (self.config.snapshots_per_min or 0) / 60, + "prime-snapshot", + run_scope(), + ) + or contextlib.nullcontext() + ): + return await run_shielded(create()) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + + async def _resume(self) -> None: + """Resume the paused box. The pause freed its memory on its node, which another + box may have taken: the platform then refuses the resume until there is room + again, so ask until it runs or `_RESUME_RETRY_SECONDS` passed.""" + deadline = time.monotonic() + _RESUME_RETRY_SECONDS + delay = 1.0 + while True: + try: + # An earlier request may have taken effect after it was given up on. + if (await self._client.get(self.info.id)).status == "RUNNING": + return + await self._client.resume(self.info.id) + await self._settle("RUNNING", "RESUME_FAILED") + return + except Exception as e: + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not resume after its " + f"memory snapshot: {e}" + ) from e + logger.warning( + "prime: sandbox %s did not resume, asking again in %.0fs: %s", + self.info.id, + delay, + e, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 15) + + def _stopped_by_snapshot(self, error: Exception) -> bool: + """Whether a call failed because a memory snapshot had stopped the box: the + call is safe to repeat once the box runs again.""" + return not self._awake.is_set() or "sandbox is paused" in str(error).lower() + + async def _settle(self, status: str, failure: str) -> None: + """Wait until the platform reports `status` for this box after a pause or a + resume (both return once accepted).""" + deadline = time.monotonic() + _PAUSE_SETTLE_SECONDS + shared = _shared_clients[asyncio.get_running_loop()] + if shared.poller is None: + shared.poller = _StatusPoller(shared.client) + while True: + try: + box = await shared.poller.next(self.info.id) + except Exception as e: + # One failed batch fails every box that waits in it: ask again. + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status}: {e}" + ) from e + await asyncio.sleep(_STATUS_POLL_SECONDS) + continue + if box.status == status: + return + if box.error_type == failure or box.status in _DEAD_STATUSES: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status}: " + f"{box.status} {box.error_type or ''} {box.error_message or ''}" + ) + if time.monotonic() > deadline: + raise SandboxError( + f"prime sandbox {self.info.id} did not reach {status} within " + f"{_PAUSE_SETTLE_SECONDS}s" + ) + + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + started = time.monotonic() + try: + durable = await self._client.wait_for_checkpoint( + snapshot.id, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) + except Exception as e: + raise SandboxError(f"prime checkpoint {snapshot.id}: {e}") from e + self.stray_snapshots.remove(snapshot) + logger.debug( + "prime: checkpoint %s durable after %.1fs on sandbox %s", + snapshot.id, + time.monotonic() - started, + self.info.id, + ) + return durable.stored_bytes + + @classmethod + async def discard(cls, snapshots: list[Snapshot]) -> list[str]: + # The platform refuses only the newest checkpoint of a running box (the parent + # of its live state); older ones go while it runs, their data kept as long as a + # newer one needs it. Usually called after teardown released `_client`. + deleted = [] + async with leased_client() as client: + for snapshot in snapshots: + try: + await client.delete_checkpoint(snapshot.id) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + if "HTTP 404" in str(e): + # Already gone: an owner that asks again must not ask forever. + deleted.append(snapshot.id) + continue + logger.warning( + "prime: failed to delete checkpoint %s: %s", snapshot.id, e + ) + else: + deleted.append(snapshot.id) + return deleted + + @classmethod + def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: + return config.model_copy( + update={"checkpoint": snapshot.id, "restore_memory": True} + ) + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: @@ -327,15 +582,14 @@ async def prepare_execution( ) async def run( - self, runtime: Runtime, argv: list[str], env: dict[str, str] + self, + runtime: Runtime, + argv: list[str], + env: dict[str, str], + *, + program: Program | None = None, ) -> ProgramResult: - async def collect(stream: AsyncIterator[bytes]) -> bytes: - tail = bytearray() - async for chunk in stream: - tail.extend(chunk[-_OUTPUT_TAIL_BYTES:]) - del tail[:-_OUTPUT_TAIL_BYTES] - return bytes(tail) - + await self._awake.wait() try: # Command sessions survive task cleanup of shared scratch directories. # Close stdin explicitly: the live-process API keeps it open by default. @@ -347,17 +601,64 @@ async def collect(stream: AsyncIterator[bytes]) -> bytes: env=runtime.process_env(env), user=runtime.user, ) - try: - stdout, stderr, exit_code = await asyncio.gather( - collect(process.stdout), collect(process.stderr), process.wait() - ) - finally: - # Closing signals and reaps a running process, then releases its stream. - await run_shielded(process.aclose()) except ( Exception ) as e: # a sandbox/API failure is one rollout's problem, not the eval's raise SandboxError(f"prime exec failed: {e}") from e + if program is not None: + program.job = {"session": process.session_uuid} + try: + return await self._collect(process) + finally: + if program is not None: + program.job = None + + async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: + try: + # The session and its process came with the snapshot; only the box is new. + process = await self._client.attach_process(self.info.id, job["session"]) + except Exception as e: + raise SandboxError(f"prime attach failed: {e}") from e + return await self._collect(process) + + async def _collect(self, process: Any) -> ProgramResult: + """Wait for `process` and keep the tail of each output stream. A memory + snapshot stops the box, which ends the stream but not the process: attach to + it again when the box runs. The output of that gap can be missing.""" + + async def collect(stream: AsyncIterator[bytes], tail: bytearray) -> None: + async for chunk in stream: + tail.extend(chunk[-_OUTPUT_TAIL_BYTES:]) + del tail[:-_OUTPUT_TAIL_BYTES] + + stdout, stderr = bytearray(), bytearray() + try: + while True: + snapshots = self._snapshots + try: + *_, exit_code = await asyncio.gather( + collect(process.stdout, stdout), + collect(process.stderr, stderr), + process.wait(), + ) + break + except Exception as e: + if snapshots == self._snapshots and not self._stopped_by_snapshot( + e + ): + raise + session = process.session_uuid + # A close would signal the process; a detach only drops the stream. + await run_shielded(process.detach()) + await self._awake.wait() + process = await self._client.attach_process(self.info.id, session) + except ( + Exception + ) as e: # a sandbox/API failure is one rollout's problem, not the eval's + raise SandboxError(f"prime exec failed: {e}") from e + finally: + # Closing signals and reaps a running process, then releases its stream. + await run_shielded(process.aclose()) return ProgramResult( exit_code=exit_code, stdout=stdout.decode(errors="replace"), @@ -416,6 +717,7 @@ async def create_and_capture_id() -> None: async def run_background( self, runtime: Runtime, argv: list[str], env: dict[str, str], log: str ) -> None: + await self._awake.wait() # A server-owned job runs without a client stream for its whole life. command = f"exec {shlex.join(argv)} > {shlex.quote(log)} 2>&1 < /dev/null" try: @@ -432,6 +734,7 @@ async def run_background( async def read( self, runtime: Runtime, path: str, max_bytes: int | None = None ) -> bytes: + await self._awake.wait() # The file API acts as the default user; a user view reads through exec. if max_bytes is not None or runtime.user is not None: reader = "cat" if max_bytes is None else f"head -c {max_bytes}" @@ -480,6 +783,7 @@ async def read( raise SandboxError(f"read {path!r}: {e}") from e async def write(self, runtime: Runtime, path: str, data: bytes) -> None: + await self._awake.wait() # The gateway creates missing parents and uploads binary data without command-line # limits. Resolve relative paths here because uploads do not use this runtime's workdir. target = ( diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 903fae63e5..7c0003d568 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -12,9 +12,11 @@ import weakref from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping +from contextvars import ContextVar from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import BaseModel from pydantic_config import BaseConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig @@ -142,6 +144,32 @@ class BaseRuntimeInfo(BaseConfig): (`Agent.run(runtime=...)`) rather than provisioning its own.""" +class Snapshot(BaseModel): + """A snapshot of a box's files and memory (`RuntimeBackend.snapshot`). A new box + made from it (`restore_config`) continues every process where the snapshot stopped + it; any number of boxes can.""" + + id: str + """The provider's id for the snapshot.""" + + +@dataclass +class Program: + """A harness's main program, as its runtime started it + (`Runtime.run(..., program=)`). A box that boots from a memory snapshot still runs + it, and `Runtime.attach` waits for it there.""" + + job: dict[str, Any] | None = None + """The provider's handle of the program while it runs; None before and after.""" + + +current_program: ContextVar[Program | None] = ContextVar( + "vf_current_program", default=None +) +"""The `Program` of the harness segment this task runs (`HarnessSession.turn`), for the +launch that runs the segment's main program.""" + + @dataclass class RuntimeCache: """Framework state for one provisioned resource, kept off the backend contract: @@ -149,6 +177,11 @@ class RuntimeCache: uv_interpreters: dict[str, str] = field(default_factory=dict) uv_locks: dict[str, asyncio.Lock] = field(default_factory=dict) + uv_ready: set[str] = field(default_factory=set) + """Users (`""` for the runtime's own) whose uv bootstrap succeeded.""" + uv_bootstrap_lock: asyncio.Lock = field(default_factory=asyncio.Lock) + """Held by a script preparation that may bootstrap uv: two bootstraps at once race + (concurrent `pip install --user`, downloads to one path, apt's lock).""" curl_ready: bool = False mcp_sources: set[str] = field(default_factory=set) mcp_lock: asyncio.Lock = field(default_factory=asyncio.Lock) @@ -186,6 +219,12 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + SUPPORTS_MEMORY_SNAPSHOTS: ClassVar[bool] = False + """Whether `snapshot` captures the box's memory with its files, so a new box + continues the processes of the old one (`restore_config`, `attach`). Rollout + checkpointing (`AgentConfig.checkpoint`) takes one at model turns, so a retry + continues from there instead of starting over.""" + info: BaseRuntimeInfo def __init__(self, name: str | None = None) -> None: @@ -194,6 +233,9 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() + self.stray_snapshots: list[Snapshot] = [] + """Snapshots this box took that never became restorable (a failed or cancelled + durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] self.stopped = False @@ -220,6 +262,43 @@ def type(self) -> str: async def start(self) -> None: pass + async def snapshot(self) -> Snapshot: + """Snapshot the box's files and memory. The box stops while the snapshot is + taken and runs again when this returns; its connections to the outside may be + dead by then. The snapshot becomes restorable only later (`wait_snapshot`).""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") + + async def attach(self, runtime: Runtime, job: dict[str, Any]) -> ProgramResult: + """Wait for a program that a box started before the memory snapshot this box + booted from (`Program.job`), and return its result like `run`.""" + raise NotImplementedError( + f"{type(self).__name__} does not support memory snapshots" + ) + + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + """Wait until `snapshot` is restorable and return its stored size in bytes, + when the provider reports it. Raises if it never becomes restorable. The box + must stay alive until then: its teardown fails a pending snapshot.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") + + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + """Free snapshots nothing will restore from and return the ids it freed. + Best-effort: a provider may refuse one a live box still needs (prime: the + newest one of a running box).""" + return await self.discard(snapshots) + + @classmethod + async def discard(cls, snapshots: list[Snapshot]) -> list[str]: + """`discard_snapshots` without a box: for an owner that holds snapshots past + the boxes that took them (`Checkpoint.pinned`).""" + return [] + + @classmethod + def restore_config(cls, config: Any, snapshot: Snapshot) -> Any: + """The runtime config for a new box that continues `snapshot` instead of + booting the task's image.""" + raise NotImplementedError(f"{cls.__name__} does not support snapshots") + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/runtimes/runtime.py b/verifiers/v1/runtimes/runtime.py index 628bc9b0cd..3f961d3988 100644 --- a/verifiers/v1/runtimes/runtime.py +++ b/verifiers/v1/runtimes/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import shlex import uuid @@ -10,13 +11,14 @@ from dataclasses import dataclass, field, replace from pathlib import PurePosixPath from types import MappingProxyType -from typing import Self +from typing import Any, Self from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import SandboxError from verifiers.v1.runtimes.base import ( _ENSURE_UV, _INSTALL_CURL, + Program, ProgramResult, RuntimeBackend, RuntimeProcess, @@ -111,10 +113,21 @@ async def set_network(self, network: NetworkPolicyConfig) -> None: ) await self.backend.prepare_execution([], policy) - async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: - """Run `argv` to completion and collect its output.""" + async def run( + self, argv: list[str], env: dict[str, str], *, program: Program | None = None + ) -> ProgramResult: + """Run `argv` to completion and collect its output. `program` marks the run as + a harness's main program: a runtime that takes memory snapshots records the + job there while it runs, for `attach`.""" + if program is not None and self.backend.SUPPORTS_MEMORY_SNAPSHOTS: + return await self.backend.run(self, argv, env, program=program) return await self.backend.run(self, argv, env) + async def attach(self, job: dict[str, Any]) -> ProgramResult: + """Wait for the job a box started before the memory snapshot this box booted + from, and return its result like `run`.""" + return await self.backend.attach(self, job) + async def exec(self, argv: list[str], env: dict[str, str]) -> RuntimeProcess: """Start `argv` and return a handle that streams its stdio.""" return await self.backend.exec(self, argv, env) @@ -179,9 +192,20 @@ async def prepare_uv_script( ) # Bootstrap dependencies before task networking is restricted; # keep task offline settings on the returned program's processes. - result = await self.run( - ["sh", "-c", command], {**(env or {}), "UV_OFFLINE": "false"} - ) + # Scripts prepare in parallel once uv is there; until then one at + # a time, so only one bootstraps it. + cache, user = self.backend.cache, self.user or "" + async with ( + contextlib.nullcontext() + if user in cache.uv_ready + else cache.uv_bootstrap_lock + ): + result = await self.run( + ["sh", "-c", command], + {**(env or {}), "UV_OFFLINE": "false"}, + ) + if result.exit_code == 0: + cache.uv_ready.add(user) if result.exit_code != 0: raise RuntimeError( "failed to prepare uv script: " diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 68d046bc0d..493464bac0 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -33,7 +33,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import HarnessError, RolloutError, TaskError from verifiers.v1.harnesses.utils.compaction import bound_tool_message -from verifiers.v1.trace import InterceptRecord, Trace +from verifiers.v1.trace import Checkpoint, InterceptRecord, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -186,11 +186,19 @@ class RolloutSession: multiplexes shares one keepalive connection pool instead of opening its own.""" proxy_mcp: MCPProxy | None = field(default=None, repr=False) """Register a host MCP backend on this session's interception server while serving.""" + reroute: Callable[[str], Awaitable[None]] | None = field(default=None, repr=False) + """Point everything this rollout reaches the interception server through at a new + base URL (the server replaced its tunnel). None when the rollout cannot move: its + tunnel loss then fails it.""" error: "RolloutError | None" = None """The latest unresolved model-call failure. The harness only sees it as an HTTP error, so when its program dies on it the rollout records this original error instead of a secondary `HarnessError`. A harness that completes cleanly after the failure handled it. Reset before each model turn, so a successful retry clears it.""" + aborted: "RolloutError | None" = None + """A request hook's failure: the task, not the provider, refused a turn. The rollout + ends the running segment with it (`Rollout.step`); the harness never sees it.""" + abort_event: asyncio.Event = field(default_factory=asyncio.Event) idempotent_requests: dict[str, IdempotentRequest] = field(default_factory=dict) """Explicit keys or marked SDK retries mapped to their replay state.""" released: bool = False @@ -219,6 +227,25 @@ class RolloutSession: last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and their parse, which the next request's equal prefix reuses instead of parsing again.""" + checkpointer: Callable[[], Awaitable[Checkpoint | None]] | None = None + """Snapshots the rollout's box as a turn goes upstream, alongside the model call; + the reply waits for the capture (`Rollout` sets it when the rollout takes + snapshots). Returns the checkpoint it took, if any.""" + + flowing: asyncio.Event = field(default_factory=asyncio.Event, repr=False) + """Cleared while a memory snapshot of the box is taken (`hold`): nothing the box + asked for is answered or written until it is set again, so the box the snapshot + stops has seen exactly what the trace held when the hold began.""" + + def __post_init__(self) -> None: + self.flowing.set() + + def hold(self) -> None: + """Stop answering the box: replies, tool verdicts and state writes wait.""" + self.flowing.clear() + + def resume(self) -> None: + self.flowing.set() @property def stopped(self) -> bool: @@ -617,6 +644,12 @@ def adopt(self, task: "asyncio.Task | None") -> None: self.tasks.add(task) task.add_done_callback(self.tasks.discard) + def abort(self, error: "RolloutError") -> None: + """End the running segment with `error`. The first abort wins.""" + if self.aborted is None: + self.aborted = error + self.abort_event.set() + def release(self) -> None: """Seal the session: no further trace mutation, and in-flight handlers cancel.""" self.released = True diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 6bcbe22179..dc919d6ed6 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.sampling import Sampling from verifiers.v1.errors import ProviderError, stop_condition from verifiers.v1.graph import RECORD_FLOAT_DECIMALS, MessageNode -from verifiers.v1.runtimes import RuntimeInfo +from verifiers.v1.runtimes import RuntimeInfo, Snapshot from verifiers.v1.semantic import ACPInfo, ParentLink, SemanticEdgeSet from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData @@ -250,6 +250,8 @@ class TimeSplit(BaseModel): class AgentSpan(TimeSpan): model: TimeSplit = Field(default_factory=TimeSplit) harness: TimeSplit = Field(default_factory=TimeSplit) + checkpoint: TimeSplit = Field(default_factory=TimeSplit) + """Time the model replies waited for checkpoint captures (`Checkpoint.held`).""" class Timing(BaseModel): @@ -266,6 +268,10 @@ class Error(BaseModel): message: str status_code: int | None = None traceback: str | None = None + attempt: int = 0 + """The run attempt that captured it: 0 for the first, then one per retry.""" + stage: str | None = None + """The rollout stage that failed (`setup`, `agent`, `finalize`, `scoring`), when known.""" class VersionInfo(BaseModel): @@ -369,6 +375,59 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's running box, memory included: the box holds the + effects of `node`'s conversation and still runs the harness program. A retry + restores it (`Trace.rewind`) into a new box that continues every process.""" + + id: str + """The snapshot's id.""" + snapshot: Snapshot + """The box's snapshot (`RuntimeBackend.snapshot`) a retry continues a new box from.""" + node: int | None = None + """The last committed node when the snapshot was taken; None before the first + turn. The restored box asks again for whatever was open after it.""" + state: Any = Field(None, exclude=True) + """The trace's typed state as the snapshot was taken (tool servers write it over + `/state`), restored with the box so a retry sees one consistent world. Live-only.""" + info: dict[str, Any] | None = Field(None, exclude=True) + """`Trace.info` as the snapshot was taken, restored with `state`. Live-only.""" + pins: dict[tuple[str, str], ToolMessage] = Field(default_factory=dict, exclude=True) + """The rollout's pinned tool results as the snapshot was taken + (`RolloutSession.pinned_tool_results`): the restored box re-sends its own copies, + which must be pinned again to match the trace. Live-only.""" + live: dict[str, Any] = Field(default_factory=dict, exclude=True) + """What continues the restored box: the harness program's job + (`HarnessSession.memory_state`), the box's relay and the secret the harness holds. + Live-only.""" + replays: dict[str, Any] = Field(default_factory=dict, exclude=True) + """The replies the server had just sent when the snapshot stopped the box + (`RolloutSession.idempotent_requests`): the restored box asks again for one that had + not arrived, and gets the reply the trace already holds. Live-only.""" + restore: dict[str, Any] | None = Field(None, exclude_if=lambda value: value is None) + """The live-only fields above as plain data, on a trace saved for another process + to continue (`verifiers.v1.restore.restorable`); None on a running rollout's + checkpoint, which holds them live.""" + time: TimeSpan = Field(default_factory=TimeSpan) + """From the snapshot request until it was durable (restorable); `end` stays 0 while + it is not.""" + captured: float = 0.0 + """When the request returned: the snapshot's content was fixed from then on.""" + held: float = 0.0 + """Seconds the turn's reply waited for the capture; 0 when the model call took + longer, since the snapshot runs alongside it.""" + bytes: int | None = None + """The snapshot's stored size, which the provider bills, once it is durable.""" + restored: bool = False + """Whether a retry restored the run from this snapshot.""" + pinned: bool = False + """Whether the rollout's owner holds the snapshot (`save_all`): no rollout deletes + it, the owner does.""" + deleted: bool = False + """Whether the snapshot was deleted (by `RolloutCheckpointConfig.keep`, or once the + run is final): it can no longer be restored.""" + + def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -594,6 +653,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry restores the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -634,6 +696,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -882,6 +952,11 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None + @property + def has_restored(self) -> bool: + """Whether a retry restored this run from a checkpoint rather than starting over.""" + return any(checkpoint.restored for checkpoint in self.checkpoints) + @property def transcript(self) -> str: """Text transcript of the last branch's messages; tool outputs are omitted.""" @@ -956,13 +1031,22 @@ def drop(self, reason: str) -> None: self.dropped = reason def split_agent_time(self) -> None: - """Split the agent span into model and harness time.""" + """Split the agent span into model, checkpoint and harness time.""" span = self.timing.agent if not span.end: return - model = sum(call.time.duration for call in self.calls) - span.model.duration = min(model, span.duration) - span.harness.duration = span.duration - span.model.duration + # A restored trace keeps the calls and checkpoints of the attempts before + # this span. + calls = sum( + call.time.duration for call in self.calls if call.time.start >= span.start + ) + # A call's span includes the time its reply waited for the capture. + held = sum(c.held for c in self.checkpoints if c.captured >= span.start) + span.checkpoint.duration = min(held, calls, span.duration) + span.model.duration = min(calls, span.duration) - span.checkpoint.duration + span.harness.duration = ( + span.duration - span.model.duration - span.checkpoint.duration + ) def record_timeout(self, stage: str) -> None: """Record a stage deadline's expiry, and stop the trace as `_timeout`. @@ -971,7 +1055,58 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) - def record_error(self, error: Exception) -> None: + def rewind_state(self, checkpoint: Checkpoint) -> None: + """Set the typed state and `info` back to `checkpoint`'s.""" + if checkpoint.state is not None: + self.state = copy.deepcopy(checkpoint.state) + if checkpoint.info is not None: + self.info = copy.deepcopy(checkpoint.info) + + def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and every call stay as the attempts' history (the + calls of the dropped nodes unlinked, so usage still counts them). Returns the + later box snapshots this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + ids = [c.id for c in self.checkpoints] + if checkpoint.id not in ids: + raise ValueError(f"checkpoint {checkpoint.id!r} is not on this trace") + del self.nodes[keep:] + for node in self.nodes: + node.semantic_parents = [ + link for link in node.semantic_parents if link.node < keep + ] + self._head_index = {} + self._pending = {} + self._generation += 1 + for call in self.calls: + if call.node is not None and call.node >= keep: + call.node = None + self.rewind_state(checkpoint) + # The restored rollout re-stamps every stage span; the run's start stays. + self.timing = Timing(start=self.timing.start) + # Checkpoints are in the order they were taken: the ones after `checkpoint` hold + # the future it rewinds past, including those at its own node. + position = ids.index(checkpoint.id) + 1 + rewound = self.checkpoints[position:] + del self.checkpoints[position:] + dropped = [c.snapshot for c in rewound if not c.pinned] + self.rewards = {} + self.metrics = {} + self.dropped = None + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + + def record_error( + self, error: Exception, *, attempt: int = 0, stage: str | None = None + ) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" self.errors.append( @@ -979,6 +1114,8 @@ def record_error(self, error: Exception) -> None: type=type(error).__name__, message=str(error), status_code=getattr(error, "status_code", None), + attempt=attempt, + stage=stage, # Provider errors already carry the actionable upstream diagnostic. # Keep full tracebacks for every other failure. traceback=None