Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ url = "https://pypi.org/simple"
default = true

[tool.uv.sources]
# Filesystem checkpoints ship in prime-cli PR #971 (unreleased): pin the SDK to that branch.
prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime.git", subdirectory = "packages/prime-sandboxes", rev = "13c8688" }
compact = { path = "environments/compact", editable = true }
glossary = { path = "environments/glossary", editable = true }
deepwiki = { path = "environments/deepwiki", editable = true }
Expand Down
72 changes: 72 additions & 0 deletions tests/v1/fixtures/checkpoint_resume_v1.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""checkpoint-resume: an agentic task whose first attempt dies mid-rollout, so the retry
resumes from the box's newest filesystem checkpoint instead of starting over.

`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model
for two file steps that build on each other, and the reward reads the second file back
out of the box. A request `@intercept` raises a `ProviderError` once the first attempt has
made two turns — after the checkpoint of turn 2 exists, whose box holds step one's file.
The resumed attempt continues from that checkpoint: it never sees the injected failure
again (the trace already carries the error), inherits step one, and finishes. Needs an
agentic (shell) harness that can continue a conversation, e.g. `bash`, on a runtime with
checkpoints (docker, prime).
"""

import uuid

import verifiers.v1 as vf
from verifiers.v1.errors import SandboxError
from verifiers.v1.runtimes import Runtime

SYSTEM = "You complete the task by running shell commands with the bash tool."
MARKER = "marker.txt"
STEP_ONE = "step1.txt"
DONE = "done.txt"
FAIL_AFTER_TURNS = 2


class CheckpointResumeTask(vf.Task[vf.TaskData]):
async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
nonce = uuid.uuid4().hex
trace.info["nonce"] = nonce
await runtime.write(MARKER, nonce.encode())

@vf.intercept
def fail_once(self, request: vf.Request, trace: vf.Trace) -> None:
# Request hooks also judge each proposed tool call after a turn commits, over
# the branch plus an empty probe result; fail only as a real turn goes upstream
# (every step's command echoes, so a real result is never empty).
last = request.messages[-1]
probe = isinstance(last, vf.ToolMessage) and not last.content
if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not trace.errors:
raise vf.ProviderError("injected mid-rollout failure", status_code=503)

@vf.reward(weight=1.0)
async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float:
try:
content = (await runtime.read(DONE)).decode(errors="replace")
except (SandboxError, OSError, ValueError):
return 0.0
return float(content.strip() == trace.info["nonce"])


class CheckpointResumeTaskset(vf.Taskset[CheckpointResumeTask, vf.TasksetConfig]):
def load(self) -> list[CheckpointResumeTask]:
return [
CheckpointResumeTask(
vf.TaskData(
idx=0,
prompt=(
"Complete these steps in order, one bash command per turn, "
"and wait for each result before the next.\n"
f"1. Run `cp {MARKER} {STEP_ONE} && echo copied`.\n"
f"2. Run `cat {STEP_ONE} > {DONE} && echo written`.\n"
"3. Reply with the single word: done."
),
system_prompt=SYSTEM,
),
self.config.task,
)
]


__all__ = ["CheckpointResumeTaskset"]
67 changes: 67 additions & 0 deletions tests/v1/test_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,73 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
assert trace.reward == 1.0


# Runtimes whose box can be checkpointed and restored: docker commits the container to
# an image, prime snapshots the sandbox filesystem (needs the platform's checkpoint API).
CHECKPOINT_RUNTIMES = [
pytest.param("docker", marks=[mark.bash, mark.docker], id="bash-harness-in-docker"),
pytest.param("prime", marks=[mark.bash, mark.prime], id="bash-harness-in-prime"),
]


@pytest.mark.e2e
@pytest.mark.parametrize("harness_runtime", CHECKPOINT_RUNTIMES, indirect=True)
async def test_retry_resumes_from_checkpoint(run_v1, harness_runtime, tmp_path):
"""A retry resumes the failed attempt's trace from its newest checkpoint: the box
restores to the state before the turn that failed, the turns up to there stay,
and the task's `setup` (a nonce the prompt never reveals) is not rerun."""
(trace,) = await run_v1(
"checkpoint-resume-v1",
harness="bash",
runtime={"type": harness_runtime},
env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}},
output_dir=tmp_path,
max_turns=8,
max_tokens=4096,
)
assert_resumed_from_checkpoint(trace)


@pytest.mark.e2e
@pytest.mark.bash
@pytest.mark.docker
async def test_retry_resumes_from_checkpoint_server(run_v1_server, tmp_path):
"""The same resume through the env-server pool: the worker's retry rewinds the
trace it is streaming, and the client still assembles the trace the worker
finished with."""
(trace,) = await run_v1_server(
"checkpoint-resume-v1",
harness="bash",
runtime={"type": "docker"},
env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}},
output_dir=tmp_path,
max_turns=8,
max_tokens=4096,
)
assert_resumed_from_checkpoint(trace)


def assert_resumed_from_checkpoint(trace) -> None:
from checkpoint_resume_v1 import FAIL_AFTER_TURNS

assert trace.ok
assert trace.reward == 1.0
assert [error.type for error in trace.errors] == ["ProviderError"]
# The failure hit as turn FAIL_AFTER_TURNS + 1 went upstream, so the newest
# checkpoint is the one taken as turn FAIL_AFTER_TURNS was asked for: the box then
# held the earlier turns' effects, and the trace kept those turns.
assert trace.agent.runtime is not None
restored_from = trace.agent.runtime.restored_from
assert restored_from is not None
assert restored_from in [checkpoint.id for checkpoint in trace.checkpoints]
assert trace.num_turns >= FAIL_AFTER_TURNS
assert len(trace.checkpoints) >= trace.num_turns
# Every turn kept has its call; the failure never went upstream, so no call
# records it.
assert [call.node for call in trace.calls] == [
i for i, node in enumerate(trace.nodes) if node.sampled
]


@pytest.mark.e2e
async def test_multi_agent_env(run_v1, tmp_path):
"""An `Env` subclass shipped with its taskset (duet-v1): two roles run the
Expand Down
39 changes: 39 additions & 0 deletions tests/v1/test_serve_delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,45 @@ async def send(data: dict) -> None:
assert len(assembly.traces[trace.id]["calls"]) == 2


@pytest.mark.asyncio
async def test_rewound_trace_streams_again_from_scratch():
"""A retry resumed from a checkpoint rewinds its trace below what the client
holds; the stream drops the client's copy and resends the trace as it now is."""
traces: list[vf.Trace] = []
frames: list[bytes] = []

async def send(data: dict) -> None:
frames.append(pack(data))

async with DeltaStreamer(lambda: traces, send) as streamer:
trace = make_trace()
traces.append(trace)
streamer.watch(trace)
add_turn(trace, "a1")
add_turn(trace, "a2")
add_turn(trace, "a3")
await settle()
assert len(assembled(frames)[trace.id]["nodes"]) == 4
trace.record_error(RuntimeError("boom"))
trace.rewind(vf.Checkpoint(id="cp", node=1))
trace.notify() # the resumed rollout's boot
await settle()
add_turn(trace, "a2 again")
await settle()
traces = [trace]
record = assembled(frames)[trace.id]
assert [n["message"]["content"] for n in record["nodes"]] == ["q", "a1", "a2 again"]
assert len(record["calls"]) == 2
assert [e["type"] for e in record["errors"]] == ["RuntimeError"]


def assembled(frames: list[bytes]) -> dict[str, dict]:
assembly = EpisodeAssembly()
for frame in frames:
assembly.apply(unpack(frame))
return assembly.traces


@pytest.mark.asyncio
async def test_pending_preview_streams_and_clears_on_commit():
traces: list[vf.Trace] = []
Expand Down
8 changes: 2 additions & 6 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions verifiers/v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
AgentInfo,
AgentSpan,
Branch,
Checkpoint,
Error,
InterceptRecord,
ModelCall,
Expand Down Expand Up @@ -226,6 +227,7 @@
"WireTaskData",
"TaskResources",
"TaskTimeout",
"Checkpoint",
"Trace",
"TraceTask",
"WireTrace",
Expand Down
80 changes: 61 additions & 19 deletions verifiers/v1/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
)
from verifiers.v1.session import RolloutLimits
from verifiers.v1.task import Task
from verifiers.v1.trace import Trace
from verifiers.v1.trace import Checkpoint, Error, Trace
from verifiers.v1.types import (
AssistantMessage,
Messages,
Expand All @@ -50,7 +50,7 @@
resolve_runtime_config,
validate_pairing,
)
from verifiers.v1.utils.retries import backoff, trace_should_retry
from verifiers.v1.utils.retries import backoff, errors_should_retry

__all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]

Expand Down Expand Up @@ -361,15 +361,19 @@ def _interception_for(
return self._server
return None

def _check_resume_support(self) -> None:
# Multi-turn capability is derived: a harness needs transcript replay, a
# native resume implementation, or a rollout-scoped session implementation.
def _supports_resume(self) -> bool:
"""Whether the harness can continue a conversation it is handed: transcript
replay, a native resume implementation, or a rollout-scoped session."""
harness = self.harness
if (
type(harness).session is Harness.session
and type(harness).resume is Harness.resume
and not harness.SUPPORTS_RESUME
):
return (
type(harness).session is not Harness.session
or type(harness).resume is not Harness.resume
or harness.SUPPORTS_RESUME
)

def _check_resume_support(self) -> None:
if not self._supports_resume():
harness = self.harness
raise ValueError(
f"Harness {harness.config.id!r} cannot host a user: resuming an "
"exchange takes transcript-backed resume (SUPPORTS_RESUME) for the "
Expand All @@ -394,35 +398,65 @@ async def run(
owner, counted in the pairing check; `on_trace` observes the trace the
moment it's minted, before any I/O. `collect_artifacts` captures the task's
declared artifacts after its finalizer while its container runtime is still
alive. Retries whole while the trace ends with a retryable error
(`config.retries`) — never into a borrowed box; the final trace keeps earlier
attempts' errors."""
alive. Retries while the trace ends with a retryable error (`config.retries`)
— never into a borrowed box. A retry resumes from the failed attempt's newest
checkpoint when it has one (`retries.checkpoint`, a runtime that supports
them, a harness that can continue a conversation), keeping that trace and its
turns up to there; otherwise it starts over. The final trace keeps every
attempt's errors."""
if self._closed:
raise RuntimeError("Agent is closed; create a new agent")
retry = self.config.retries
history: list = []
can_resume = retry.checkpoint and runtime is None and self._supports_resume()
history: list[Error] = [] # errors of the attempts started over from
trace: Trace | None = None
resume: Checkpoint | None = None
for attempt in range(retry.max_retries + 1):
resumed = trace if resume is not None else None
seen = len(resumed.errors) if resumed is not None else 0

def should_retry(
current: Trace, *, attempt: int = attempt, seen: int = seen
) -> bool:
# This attempt's own errors decide — a resumed trace still carries
# the earlier attempts'.
return attempt < retry.max_retries and errors_should_retry(
current.errors[seen:], retry
)

trace = await self._run_once(
task, runtime, tools, on_trace, collect_artifacts
task,
runtime,
tools,
on_trace,
collect_artifacts,
resume=(resumed, resume) if resumed is not None else None,
retry=should_retry if can_resume and retry.max_retries else None,
)
if attempt == retry.max_retries or not trace_should_retry(trace, retry):
if not should_retry(trace):
break
if runtime is not None:
logger.warning(
"not retrying the rollout on a borrowed box (its state is no "
"longer the task's start state); the error stands"
)
break
history.extend(trace.errors)
resume = trace.checkpoints[-1] if can_resume and trace.checkpoints else None
if resume is None:
history.extend(trace.errors)
delay = backoff(attempt)
logger.warning(
"retrying agent rollout (retry %d/%d) in %.1fs after error: %s",
"retrying agent rollout (retry %d/%d) in %.1fs after error: %s (%s)",
attempt + 1,
retry.max_retries,
delay,
trace.last_error.type if trace.last_error else "?",
f"resuming from checkpoint {resume.id}"
if resume is not None
else "from scratch",
)
await asyncio.sleep(delay)
assert trace is not None
if history:
# The full history rides the final trace either way; success is the
# `ok` stamp, never errors-emptiness.
Expand All @@ -436,6 +470,9 @@ async def _run_once(
shared_tools: Mapping[str, SharedToolServer] | None,
on_trace: Callable[[Trace], None] | None,
collect_artifacts: bool,
*,
resume: tuple[Trace, Checkpoint] | None = None,
retry: Callable[[Trace], bool] | None = None,
) -> Trace:
params = self._rollout_params(task, runtime, dict(shared_tools or {}))
if collect_artifacts and isinstance(params["runtime_config"], SubprocessConfig):
Expand All @@ -447,11 +484,16 @@ async def _run_once(
task=task,
on_trace=on_trace,
collect_artifacts=collect_artifacts,
resume=resume,
retry=retry,
**params,
)
try:
if await run.open():
await run.step()
if resume is not None:
await run.resume()
else:
await run.step()
if run.ok:
run.trace.stop("agent_completed")
trace = await run.close()
Expand Down
7 changes: 7 additions & 0 deletions verifiers/v1/configs/retries.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,10 @@ class RetryConfig(BaseConfig):
"""Only retry errors whose type is listed. Empty = retry anything not excluded."""
exclude: list[str] = Field(default_factory=list)
"""Never retry errors whose type is listed (wins over `include`)."""
checkpoint: bool = True
"""Checkpoint the runtime's filesystem before every model turn (on runtimes that
support it, e.g. prime and docker) and resume a retry from the newest checkpoint:
the trace keeps its turns up to there and the harness continues the conversation
in a box restored to that state. False retries from scratch. Only the box is
restored — state kept outside it (a tool server's typed rollout state) stays as
the failed attempt left it."""
Loading
Loading