Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
357c360
feat: keep rollouts through a lost interception tunnel
mikasenghaas Nov 5, 2026
d81764f
always relay a remote harness and fix review findings
mikasenghaas Nov 5, 2026
cbd8410
start the relay alongside the harness setup
mikasenghaas Nov 5, 2026
c05642f
simplify the relay wiring and trust certifi's roots
mikasenghaas Nov 6, 2026
a2a2936
fail the rollout when its relay does not start
mikasenghaas Nov 6, 2026
7b3f706
name the relay in the trace when it fails
mikasenghaas Nov 6, 2026
1c62100
set the relay field on the rollout test stub
mikasenghaas Nov 6, 2026
d72d875
bootstrap uv once per box at a time
mikasenghaas Nov 6, 2026
19974a4
fix relay review findings
mikasenghaas Nov 6, 2026
31d658b
name RelayError in the relay docs
mikasenghaas Nov 6, 2026
01f4f57
feat: restore rollouts from filesystem checkpoints
mikasenghaas Nov 4, 2026
824ca30
narrow checkpoints to retries and name snapshots
mikasenghaas Nov 6, 2026
7a5330a
record cancelled turns and shorten checkpoint docs
mikasenghaas Nov 6, 2026
03a52bd
add the checkpoint restore integration test back
mikasenghaas Nov 7, 2026
94cf961
keep uv.lock as on the branch
mikasenghaas Nov 7, 2026
e042091
drop box-less checkpoints
mikasenghaas Nov 7, 2026
fcf1800
check that a checkpointed run leaks no snapshot
mikasenghaas Nov 7, 2026
d3043e2
Merge main into feat/fs-rollout-ckpt
mikasenghaas Nov 7, 2026
9f04c75
Merge main into feat/fs-rollout-ckpt
mikasenghaas Nov 7, 2026
d77caf0
join a cancelled harness turn through cancellation
mikasenghaas Nov 7, 2026
3a581f8
snapshot from turn n and drop after, all and type
mikasenghaas Nov 9, 2026
8b20e18
derive restorable harnesses and step back after a dead restore
mikasenghaas Nov 10, 2026
014d638
fix the dead-restore check's grouping
mikasenghaas Nov 10, 2026
07d4a31
Merge remote-tracking branch 'origin/feat/fs-rollout-ckpt' into feat/…
mikasenghaas Nov 10, 2026
9c150ea
Merge remote-tracking branch 'origin/feat/interception-relay' into fe…
mikasenghaas Nov 10, 2026
10e4670
restore rollout retries from memory snapshots
mikasenghaas Nov 10, 2026
7a520d9
save running rollouts as restore points for another process
mikasenghaas Nov 10, 2026
14fa956
test a saved rollout continued by another owner
mikasenghaas Nov 10, 2026
addc9d7
share batched status polls between pausing boxes
mikasenghaas Nov 10, 2026
8ba38c8
test and document memory checkpoints
mikasenghaas Nov 10, 2026
221b54d
fix lint
mikasenghaas Nov 10, 2026
6dd57f2
let an owner save rollouts whatever snapshot their turns take
mikasenghaas Nov 10, 2026
87e35c5
count a snapshot that is already gone as deleted
mikasenghaas Nov 10, 2026
562b10c
skip the restore check for a harness that survives the failed reply
mikasenghaas Nov 10, 2026
d10b76d
say why a running rollout takes no restore point
mikasenghaas Nov 10, 2026
d85c50b
return a saved rollout's restore point before its snapshot is durable
mikasenghaas Nov 10, 2026
bd383bb
ask again when a box does not resume after its memory snapshot
mikasenghaas Nov 10, 2026
a1ca7b9
check the box before asking it to resume again
mikasenghaas Nov 10, 2026
0c8eab6
start over when a restore point leads nowhere
mikasenghaas Nov 10, 2026
569389f
take memory snapshots only
mikasenghaas Nov 10, 2026
273e169
Merge remote-tracking branch 'origin/main' into feat/mem-ckpt
mikasenghaas Nov 10, 2026
3b4db58
snapshot a rollout when its model call fails, not at turns
mikasenghaas Nov 10, 2026
c6c3854
sort imports
mikasenghaas Nov 10, 2026
c85ae3e
test the restore of a box with two running sub-agents
mikasenghaas Nov 11, 2026
496124e
install the relay before the harness setup starts
mikasenghaas Nov 11, 2026
ae50ebe
install the relay before the harness setup starts
mikasenghaas Nov 11, 2026
2e0c806
pace memory snapshots run-wide
mikasenghaas Nov 11, 2026
ad0eb22
continue a saved rollout from its trace alone
mikasenghaas Nov 11, 2026
89685a4
Merge branch 'main' into feat/mem-ckpt
mikasenghaas Nov 13, 2026
c16809f
run the harness program in a session on prime
mikasenghaas Nov 13, 2026
a1dd2ed
remove the snapshot on a failed model call
mikasenghaas Nov 13, 2026
6d2e9ea
Merge branch 'feat/mem-ckpt' into feat/mem-ckpt-sessions
mikasenghaas Nov 13, 2026
93ff84b
test that every saved turn continues to the same result
mikasenghaas Nov 13, 2026
71d2406
Merge branch 'feat/mem-ckpt-sessions' into feat/mem-ckpt
mikasenghaas Nov 13, 2026
fa84fa5
keep a box running when the wait for its pause fails
mikasenghaas Nov 13, 2026
a1b88d8
tell the server's own 404 from a tunnel edge's page
mikasenghaas Nov 13, 2026
2537fb0
keep a hold and a snapshot from undoing each other
mikasenghaas Nov 13, 2026
78a8436
pin prime-sandboxes to the attach branch
mikasenghaas Nov 13, 2026
0486bce
stop the box in the cancelled-save test
mikasenghaas Nov 13, 2026
a05feb5
Merge remote-tracking branch 'origin/main' into feat/interception-relay
mikasenghaas Nov 13, 2026
167b4b6
tell the server's own 404 from a tunnel edge's page
mikasenghaas Nov 13, 2026
bc68020
Merge branch 'feat/interception-relay' into feat/mem-ckpt
mikasenghaas Nov 13, 2026
3be0b89
probe which harnesses echo their replies
mikasenghaas Nov 13, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 0 additions & 36 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,54 +81,18 @@ jobs:
max-parallel: 16
matrix:
include:
- behavior: Single-Turn
tests: tests/integration/test_single_turn.py
- behavior: Multi-Turn
tests: tests/integration/test_multi_turn.py
- behavior: Multi-Turn Tool
tests: tests/integration/test_multi_turn_tool.py
- behavior: Stateful Tool
tests: tests/integration/test_stateful_tool.py
- behavior: Shared Tool
tests: tests/integration/test_shared_tool.py
- behavior: Multimodal Tool
tests: tests/integration/test_multimodal_tool.py
- behavior: User Echo
tests: tests/integration/test_user_echo.py
- behavior: Resume Tool
tests: tests/integration/test_resume_tool.py
- behavior: User Sim
tests: tests/integration/test_user_sim.py
- behavior: Agent User
tests: tests/integration/test_agent_user.py
- behavior: Network Policy
tests: tests/integration/test_network_policy.py
- behavior: Service VMs
tests: tests/integration/test_service_scoring.py
- behavior: Native Compose
tests: tests/integration/test_native_compose.py
- behavior: Tool VMs
tests: tests/integration/test_tool_vms.py
- behavior: Mounts
tests: tests/integration/test_mounts.py
- behavior: Sampling
tests: tests/integration/test_sampling.py
- behavior: Parallel
tests: tests/integration/test_parallel.py
- behavior: Scoring Network
tests: tests/integration/test_scoring_network.py
- behavior: Skills
tests: tests/integration/test_skills.py
- behavior: Limits
tests: tests/integration/test_limits.py
- behavior: Rubric Judge
tests: tests/integration/test_rubric_judge.py
- behavior: Agentic Judge
tests: tests/integration/test_agentic_judge.py
- behavior: Multi-Agent
tests: tests/integration/test_multi_agent.py
- behavior: Examples
tests: tests/integration/test_examples.py
env:
PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }}
PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }}
Expand Down
29 changes: 29 additions & 0 deletions docs/agent.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,32 @@ max_turn_tokens = 4096
skip_on_truncated = true
skip_on_timeout = true
```

## Retries and checkpoints

`[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget.

Rollout checkpointing is on by default: a rollout can be snapshotted, and a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over.

By default a rollout takes a snapshot only when its owner saves it (`save_all`, below). `interval` adds snapshots that the rollout takes by itself, at every n-th model turn. This covers failures that leave no box to snapshot: the box died, or the harness program crashed.

Without a snapshot, a retry starts over.

A snapshot holds the box's files and its memory. The restored box continues every process where the snapshot stopped it: the harness program, its sub-agents and shells, and the task's own processes. The harness gets no notice and starts no new process; a model call that was open is asked again. The snapshot stops the box for a few seconds.

A rollout takes snapshots only when all of these hold. Otherwise a retry starts over, with a warning once per harness and runtime:

- The runtime takes memory snapshots (`SUPPORTS_MEMORY_SNAPSHOTS`: prime).
- The harness runs detached from the host (`SUPPORTS_MEMORY_RESTORE`: null, bash and the ACP harnesses).
- The rollout runs in a box of its own, not in a borrowed one.
- The rollout has no tool servers, no shared tools and no services of its own. They reach the host at an address that does not follow the box to its restore.

A retry also starts over when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off.

```toml
[env.agent.checkpoint]
# interval = 10 # snapshot at the 10th, 20th, ... model turn (off by default)
keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest
```

An owner that outlives its process, such as a training run that stops and resumes, saves its running rollouts instead of waiting for a failure. `verifiers.v1.restore.save_all()` takes a memory snapshot of every running rollout of the process now and returns the trace of each as plain data the owner stores. The trace holds everything a restore needs, on the entry of its checkpoint. A later process continues a rollout from its saved trace with `RunSlot(task, id=..., restore=trace)` (or `Agent.run(task, restore=trace)`): same trace, a new box, and the harness program still running. The owner holds these snapshots; it deletes them with `PrimeBackend.discard` once nothing will restore them. A saved trace continues one rollout: an episode with more than one agent run cannot continue from the trace of one of them, and starts over.
11 changes: 11 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,17 @@ Routing traffic through the interception server enables several things beyond fo
- Applying sampling parameters in harnesses that don't expose those settings directly
- Running the task's [stop and interception hooks](tasksets.md#stops-and-interception) to inspect or rewrite tool responses and server-side web search results (for example, to block reward hacks)

### Relay

A Prime tunnel can drop for a while (its client reconnects) or for good (the tunnel service drops its registration). A harness in a remote box therefore reaches the interception server through a relay in its own box, at `127.0.0.1`, not at the tunnel URL:

- While the tunnel cannot be reached, the relay retries the model call for up to 5 minutes, so the harness does not see a tunnel that reconnects.
- When a tunnel is lost for good, its interception server opens a new tunnel and points each relay at the new URL. A model call that was in flight goes through the new tunnel; the server joins it to the call it already serves, so the trace gets no second turn.

A box restored from a memory snapshot (`agent.checkpoint`) still runs its relay. The new rollout points that relay at its own interception server and maps the secret the harness holds to the new session's secret, so the harness keeps its URL and its secret. Model calls that were open when the snapshot stopped the box go to the new server.

A rollout with tool servers or deployment services still fails on a lost tunnel, because they reach the interception server at the tunnel URL. The relay is a uv script, like the harness programs and tool servers verifiers already runs in the box. A relay that does not start, gives up on a request or exits fails the rollout with a `RelayError`.

## Finishing a rollout

When the agent finishes its turn normally, the rollout collects artifacts and runs in-place grading before closing the harness session:
Expand Down
1 change: 1 addition & 0 deletions docs/harnesses.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ Class-level capability flags declare which optional features your harness implem
- `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead.
- `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program.
- `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)).
- `SUPPORTS_MEMORY_RESTORE`: indicates that a box restored from a memory snapshot continues the harness (`agent.checkpoint`). The program must run as a job of the box that the host only waits for (`runtime.run(..., program=current_program.get())`), keep no connection to the host open, and reach the host only through the endpoint it was given: the box's relay follows a restored box to its new interception server.
- `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them.
- `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them.

Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ url = "https://pypi.org/simple"
default = true

[tool.uv.sources]
# Temporary, until a release has `attach_process` (PrimeIntellect-ai/prime#1147).
prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime", rev = "fae6d0bc91717b95ff1b109d93d58a22e3408df2", subdirectory = "packages/prime-sandboxes" }
compact = { path = "environments/compact", editable = true }
glossary = { path = "environments/glossary", editable = true }
deepwiki = { path = "environments/deepwiki", editable = true }
Expand Down
44 changes: 43 additions & 1 deletion tests/integration/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,46 @@ def _eval_config(
)


def _echo_probe(taskset: str, episodes) -> None:
"""Experiment: does the harness show each reply again in its next request, in a form
the graph matches? A sampled node with a child was matched. A sampled node without
one is a final reply, or a reply whose echo did not match: then its parent has
another child that is an assistant message the model did not sample."""
import warnings

for episode in episodes:
for trace in episode.traces:
nodes = trace.nodes
children: dict[int | None, list[int]] = {}
for index, node in enumerate(nodes):
children.setdefault(node.parent, []).append(index)
sampled = [i for i, node in enumerate(nodes) if node.sampled]
matched = [i for i in sampled if children.get(i)]
leaves = [i for i in sampled if not children.get(i)]
lost = [
i
for i in leaves
if any(
j != i
and not nodes[j].sampled
and nodes[j].message.role == "assistant"
for j in children.get(nodes[i].parent, [])
)
]
unsampled_assistant = sum(
not node.sampled and node.message.role == "assistant" for node in nodes
)
harness = trace.agent.config.harness
warnings.warn(
f"ECHO harness={harness.id if harness else None} taskset={taskset} "
f"dialect={getattr(harness, 'dialect', None)} sampled={len(sampled)} "
f"matched={len(matched)} leaves={len(leaves)} lost={len(lost)} "
f"unsampled_assistant={unsampled_assistant} "
f"branches={trace.num_branches} nodes={len(nodes)} ok={trace.ok}",
stacklevel=1,
)


@pytest.fixture
def run(tmp_path):
"""Run a taskset's episodes in-process with `vf-run`'s runner, saving them under
Expand All @@ -185,7 +225,9 @@ def run(tmp_path):
async def _episodes(taskset: str, **kwargs) -> list[Episode]:
config = _eval_config(taskset, **kwargs)
config.output_dir = tmp_path
return await run_episodes(config)
episodes = await run_episodes(config)
_echo_probe(taskset, episodes)
return episodes

return _episodes

Expand Down
98 changes: 98 additions & 0 deletions tests/integration/fixtures/memory_restore_v1.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
"""memory-restore: a task whose first attempt dies mid-rollout, so the retry restores the
box's newest memory snapshot and continues the harness program the box still runs.

`setup` starts a process in the box that holds a nonce only in its memory and hands it
to whoever reads `nonce.fifo`. The nonce is on no disk, so only a box that continues
the process from a memory snapshot can answer.
The reward reads the nonce from the process. The task asks a harness with a shell for
two commands that build on each other, and a metric checks the file they leave.

A response `@intercept` raises a `ProviderError` at the model turn `fail_turn`, for
every call of the first attempt: the harness gets the error as its reply and dies,
after the snapshot of that turn exists. The restored box still waits for that turn's
reply, gets it from the retry, and finishes: no new harness process, no notice to the
model. One failed call is not enough: the snapshot resets the box's connections, its
relay asks again, and a call that then succeeds would let the harness continue.
"""

import uuid

import verifiers.v1 as vf
from verifiers.v1.errors import SandboxError
from verifiers.v1.runtimes import Runtime

FIFO = "nonce.fifo"
STEP_ONE = "step1.txt"
DONE = "done.txt"
READ = f"timeout 10 cat {FIFO}"
# The nonce is the script's first argument: it lives in the process, not in a file. The
# sleep lets a reader see the end of one copy before the next write opens the FIFO.
SERVER = (
f'rm -f {FIFO}; mkfifo {FIFO}; while :; do printf %s "$1" > {FIFO}; sleep 0.5; done'
)


class MemoryRestoreTaskConfig(vf.TaskConfig):
fail_turn: int = 2
"""The model turn whose reply fails once (from 1)."""


class MemoryRestoreConfig(vf.TasksetConfig):
task: MemoryRestoreTaskConfig = MemoryRestoreTaskConfig()


class MemoryRestoreTask(vf.Task[vf.TaskData, vf.State, MemoryRestoreTaskConfig]):
async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
nonce = uuid.uuid4().hex
trace.info["nonce"] = nonce
await runtime.run_background(
["sh", "-c", SERVER, "sh", nonce], {}, "server.log"
)

@vf.intercept
def fail_once(self, response: vf.Response, trace: vf.Trace) -> None:
if trace.attempt == 1 and trace.num_turns + 1 >= self.config.fail_turn:
# A 4xx: harness SDKs end the turn at once instead of retrying it until the
# rollout's deadline.
raise vf.ProviderError("injected mid-rollout failure", status_code=400)

async def _read(self, runtime: Runtime, command: str) -> str:
try:
result = await runtime.run(["sh", "-c", command], {})
except (SandboxError, OSError):
return ""
return result.stdout.strip()

@vf.reward(weight=1.0)
async def process_alive(self, trace: vf.Trace, runtime: Runtime) -> float:
"""The process `setup` started still runs and still holds the nonce."""
return float(await self._read(runtime, READ) == trace.info["nonce"])

@vf.metric
async def copied(self, trace: vf.Trace, runtime: Runtime) -> float:
"""The agent's two commands left the nonce in the done file."""
return float(await self._read(runtime, f"cat {DONE}") == trace.info["nonce"])


class MemoryRestoreTaskset(vf.Taskset[MemoryRestoreTask, MemoryRestoreConfig]):
def load(self) -> list[MemoryRestoreTask]:
return [
MemoryRestoreTask(
vf.TaskData(
idx=0,
prompt=(
"Run these shell commands in the working directory, one "
"command per turn, and wait for each result before the "
"next. If you cannot run commands, reply with the single "
"word: done.\n"
f"1. `{READ} > {STEP_ONE}`\n"
f"2. `cp {STEP_ONE} {DONE}`\n"
"3. Reply with the single word: done."
),
),
self.config.task,
)
]


__all__ = ["MemoryRestoreTaskset"]
108 changes: 108 additions & 0 deletions tests/integration/fixtures/subagent_restore.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""subagent-restore: a task whose agent hands two jobs to two sub-agents that run at the
same time, so a box snapshotted in the middle holds two running sub-agents.

`setup` starts one process per sub-agent that holds a nonce only in its memory and
hands it to whoever reads its FIFO. Each sub-agent reads the nonce twice, one command
per turn. The second command first waits for the other sub-agent's first file and then
sleeps, so from the moment both sub-agents have a turn on the trace, neither finishes
for `HOLD_SECONDS`: a snapshot taken then catches both at work. The second read comes
from the process, so it works only in a box that kept its memory.
"""

import uuid

import verifiers.v1 as vf
from verifiers.v1.errors import SandboxError
from verifiers.v1.runtimes import Runtime

NAMES = ("alpha", "bravo")
ROOT_MARK = "COORDINATOR-TASK"
HOLD_SECONDS = 20
SERVER = 'rm -f "$2"; mkfifo "$2"; while :; do printf %s "$1" > "$2"; sleep 0.5; done'


def mark(name: str) -> str:
"""The text only `name`'s brief holds."""
return f"SUBAGENT-{name.upper()}"


def brief(name: str) -> str:
other = NAMES[1 - NAMES.index(name)]
return (
f"{mark(name)}. Run these shell commands in the working directory, one "
"command per turn, and wait for each result before the next.\n"
f"1. `timeout 10 cat {name}.fifo > {name}.1`\n"
f"2. `timeout 120 sh -c 'until [ -f {other}.1 ]; do sleep 1; done'; "
f"sleep {HOLD_SECONDS}; timeout 10 cat {name}.fifo > {name}.done`\n"
"3. Reply with the single word: done."
)


def sub_agent_turns(trace: vf.Trace) -> dict[str, list[int]]:
"""The sampled nodes of each sub-agent: those whose history holds the sub-agent's
brief and not the coordinator's task."""
turns: dict[str, list[int]] = {name: [] for name in NAMES}
for index, node in enumerate(trace.nodes):
if not node.sampled:
continue
history, cursor = "", node.parent
while cursor is not None:
history += str(trace.nodes[cursor].message.model_dump())
cursor = trace.nodes[cursor].parent
if ROOT_MARK in history:
continue
for name in NAMES:
if mark(name) in history:
turns[name].append(index)
return turns


class SubagentRestoreTask(vf.Task):
async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
for name in NAMES:
nonce = uuid.uuid4().hex
trace.info[name] = nonce
await runtime.run_background(
["sh", "-c", SERVER, "sh", nonce, f"{name}.fifo"], {}, f"{name}.log"
)

async def _read(self, runtime: Runtime, command: str) -> str:
try:
result = await runtime.run(["sh", "-c", command], {})
except (SandboxError, OSError):
return ""
return result.stdout.strip()

@vf.reward(weight=1.0)
async def both_done(self, trace: vf.Trace, runtime: Runtime) -> float:
"""Each sub-agent's second read left its nonce in its done file."""
done = [
await self._read(runtime, f"cat {name}.done") == trace.info[name]
for name in NAMES
]
return float(all(done))


class SubagentRestoreTaskset(vf.Taskset[SubagentRestoreTask, vf.TasksetConfig]):
def load(self) -> list[SubagentRestoreTask]:
briefs = "\n\n".join(
f"Brief for sub-agent {i + 1}:\n{brief(name)}"
for i, name in enumerate(NAMES)
)
return [
SubagentRestoreTask(
vf.TaskData(
idx=0,
prompt=(
f"{ROOT_MARK}. Do not run the commands below yourself. Start "
"exactly two sub-agents so that both run at the same time, "
"and give each one its brief word for word. Wait for both, "
"then reply with the single word: done.\n\n"
f"{briefs}"
),
)
)
]


__all__ = ["SubagentRestoreTaskset"]
Loading
Loading