Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion docs/v1/evaluation.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,21 @@ The output from evaluations are written into `outputs/<env>--<model>--<harness>/

## Resuming evaluations

`--resume <output-dir>` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone.
`--resume <output-dir>` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. Rollouts the interruption cut off run again from the start; prime-rl's `uv run eval --resume` replays them from their live traces instead.

## Replaying traces

`vf.Replay` answers model calls with the replies in recorded traces. The harness runs every replayed tool call again, so it rebuilds the recorded rollout's files, processes, and tool state. It has two modes:

- `resume` (default): replay while each prompt equals its recorded prompt, then sample live. This continues an interrupted rollout.
- `playback`: replay every recorded reply whatever the prompt, never sample live, and stop with `replay_ended` when the recording runs out. This reproduces a trace's deliverables, or drives one seat of a multi-agent env.

```python
async with vf.Agent(config, replay=vf.Replay(traces)) as agent:
trace = await agent.run(task)
```

An env plays back one seat from `setup()`: `agents.solver.ctx = dataclasses.replace(agents.solver.ctx, replay=vf.Replay(traces, mode="playback"))`. `ModelCall.replayed` tags each replayed call: `exact` when its prompt equals the recorded prompt, `forced` when a playback serves it to a prompt that differs. Tool calls run again, side effects included. Replay is for evaluation only: the train client needs token ids.

## Disabling tools

Expand Down
2 changes: 2 additions & 0 deletions verifiers/v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
Toolset,
ToolsetConfig,
)
from verifiers.v1.replay import Replay
from verifiers.v1.runtimes import (
ApptainerConfig,
DockerConfig,
Expand Down Expand Up @@ -229,6 +230,7 @@
"Trace",
"TraceTask",
"WireTrace",
"Replay",
"Reward",
"EnvInfo",
"Episode",
Expand Down
10 changes: 8 additions & 2 deletions verifiers/v1/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from verifiers.v1.harness import Harness
from verifiers.v1.interception import Interception, InterceptionServer
from verifiers.v1.mcp import SharedToolServer
from verifiers.v1.replay import Replay
from verifiers.v1.rollout import Rollout, RolloutTimeouts
from verifiers.v1.runtimes import (
Runtime,
Expand Down Expand Up @@ -268,6 +269,8 @@ def __init__(
config: AgentConfig,
*,
interception: Interception | None = None,
name: str = "agent",
replay: Replay | None = None,
) -> None:
from verifiers.v1.utils.loaders import harness_config_type, load_harness

Expand All @@ -286,10 +289,12 @@ def __init__(
config = config.model_copy(update={"sampling": Sampling()})
self.config = config
self.harness = load_harness(config.harness)
self._name = name
self.ctx = ModelContext(
model=config.model,
client=config.client or EvalClientConfig(),
sampling=config.sampling,
replay=replay,
)
self._closed = False
self.runtime_config: RuntimeConfig = config.runtime
Expand Down Expand Up @@ -562,6 +567,7 @@ def _rollout_params(
timeouts = resolve_rollout_timeouts(self.timeout, task)
return {
"agent_config": self.config,
"agent_name": self._name,
"harness": harness,
"ctx": self.ctx,
"runtime_config": runtime_config,
Expand Down Expand Up @@ -612,11 +618,11 @@ def __init__(
on_trace: Callable[[Trace], None] | None,
on_discard: Callable[[Trace], None] | None,
warned_resources: set,
replay: Replay | None = None,
) -> None:
super().__init__(config, interception=interception)
super().__init__(config, interception=interception, name=name, replay=replay)
# Resource warnings dedupe env-wide, not per episode.
self._warned_resources = warned_resources
self._name = name
self._shared_tools = shared_tools
self._task_cls = task_cls
self._gate = gate
Expand Down
6 changes: 6 additions & 0 deletions verifiers/v1/clients/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING

from verifiers.v1.configs.client import (
BaseClientConfig,
Expand All @@ -13,6 +14,9 @@
from verifiers.v1.graph import PendingTurn
from verifiers.v1.types import Response, Sampling, SamplingConfig

if TYPE_CHECKING:
from verifiers.v1.replay import Replay

SESSION_ID_HEADER = "X-Session-ID"
"""Per-rollout routing header (the trace id, same value every turn), so a session-affinity
router pins a rollout's turns to one engine and its growing prefix stays KV-cached."""
Expand Down Expand Up @@ -87,3 +91,5 @@ class ModelContext:
model: str
client: ClientConfig
sampling: Sampling = field(default_factory=Sampling)
replay: "Replay | None" = field(default=None, compare=False)
"""Recorded traces whose replies answer matching requests before the live model."""
1 change: 1 addition & 0 deletions verifiers/v1/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ def make(name: str, spec: AgentConfig) -> Agent:
on_trace=on_trace,
on_discard=on_discard,
warned_resources=self._warned_resources,
replay=ctx.replay,
)

agents = Agents(self.config, make)
Expand Down
48 changes: 45 additions & 3 deletions verifiers/v1/interception/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
from verifiers.v1.semantic import ACPInfo, extract_acp_info
from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession
from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan
from verifiers.v1.types import FinishReason, Response, Usage
from verifiers.v1.types import FinishReason, Request, Response, Usage

logger = logging.getLogger(__name__)

Expand All @@ -93,6 +93,29 @@
IDEMPOTENCY_CACHE_MAX_COMPLETED = 64


def _replay_reply(
session: RolloutSession, dialect: Dialect, request: Request
) -> tuple[Response, Literal["exact", "forced"]] | None:
"""The recorded reply that answers this request, rendered for the harness; None when
there is none, or when the lookup fails (logged)."""
if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig):
return None
try:
replayed = session.ctx.replay.take(
session.trace.id, request.messages, request.tools
)
if replayed is not None:
replayed[0].raw = dialect.response_to_wire(replayed[0], session.ctx.model)
return replayed
except Exception:
logger.warning(
"replay lookup failed, sampling live: id=%s",
session.trace.id,
exc_info=True,
)
return None


def is_retried_request(headers: Mapping[str, str]) -> bool:
try:
return int(headers.get(RETRY_COUNT_HEADER, 0)) > 0
Expand Down Expand Up @@ -528,6 +551,7 @@ def record_call(
error: BaseException | None = None,
policy_paths: list[str] | None = None,
acp: ACPInfo | None = None,
replayed: Literal["exact", "forced"] | None = None,
) -> None:
"""Append one provider exchange to the trace's per-call records (`Trace.calls`):
the model + effective settings that went upstream, timing, and — when the call
Expand Down Expand Up @@ -576,6 +600,7 @@ def record_call(
if policy_paths
else None,
acp=acp,
replayed=replayed,
)
)
session.trace.notify()
Expand Down Expand Up @@ -777,11 +802,25 @@ async def sample() -> web.Response:
node: int | None = None
error: Exception | None = None
started = time.time()
replayed = _replay_reply(session, dialect, model_request)
if (
replayed is None
and session.ctx.replay is not None
and session.ctx.replay.plays_back(session.trace.id)
):
# A playback never samples live: its recording ended here.
turn.commit_prompt()
session.trace.stop("replay_ended")
return web.json_response(
dialect.error_body("rollout stopped: replay_ended"), status=400
)
try:
try:
# What actually goes upstream: the native body with the rollout's model +
# sampling imposed — recorded raw on the trace, per call.
if relay:
if replayed is not None:
call_response = replayed[0]
elif relay:
reply = await session.client.relay(
dialect,
body,
Expand Down Expand Up @@ -809,7 +848,9 @@ async def sample() -> web.Response:
)
response_rewrites = []
stopped = None
if session.response_interceptors or session.response_stops:
if replayed is None and (
session.response_interceptors or session.response_stops
):
(
call_response,
response_rewrites,
Expand Down Expand Up @@ -891,6 +932,7 @@ async def sample() -> web.Response:
error=error,
policy_paths=policy_paths,
acp=acp,
replayed=replayed[1] if replayed is not None else None,
)
return serve(call_response, events)

Expand Down
151 changes: 151 additions & 0 deletions verifiers/v1/replay.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
"""Replay recorded model replies, in one of two modes.

`resume` (the default) continues an interrupted rollout: it replays recorded replies while
each prompt is exactly its recorded prompt (`exact`), then samples live from the first
prompt that differs.

`playback` plays a recording back, on any task: it replays every recorded reply whatever the
tools returned or other agents said, never samples live, and stops the rollout
(`replay_ended`) when the recording runs out. A reply played onto a prompt that differs from
its recorded prompt is `forced`: the recorded model never saw that prompt.

A rollout whose context carries a `Replay` claims a free recording of its agent (on its task,
in resume) and runs under the recording's trace id. A failed attempt frees its recording, so
the attempt that retries it replays the recording again, under its own id. Each model
request is answered with the recorded reply that followed the prompt's last assistant
message, or that opened a branch when the prompt has none. The harness still runs every tool
call, so its runtime and program state are rebuilt."""

import logging
from dataclasses import dataclass, field
from typing import Literal

from verifiers.v1 import graph
from verifiers.v1.trace import Trace
from verifiers.v1.types import AssistantMessage, Message, Response, Tool

logger = logging.getLogger(__name__)

ReplayMode = Literal["resume", "playback"]
ReplayMatch = Literal["exact", "forced"]


@dataclass(eq=False)
class _Recording:
trace: Trace
adopted: bool = False
used: set[int] = field(default_factory=set)
hashes: list[str] = field(default_factory=list)
replies: dict[str | None, list[int]] = field(default_factory=dict)
"""Recorded replies by the hash of the assistant message their prompt ends after."""

def __post_init__(self) -> None:
nodes = self.trace.nodes
self.hashes = [graph.message_hash(node.message) for node in nodes]
for reply, node in enumerate(nodes):
if not node.sampled:
continue
last = node.parent
while last is not None and not isinstance(
nodes[last].message, AssistantMessage
):
last = nodes[last].parent
key = None if last is None else self.hashes[last]
self.replies.setdefault(key, []).append(reply)

def is_prompt(self, reply: int, hashes: list[str], tools: list[Tool]) -> bool:
"""Whether a prompt with these message hashes and tools is `reply`'s recorded prompt."""
nodes: list[int] = []
current = self.trace.nodes[reply].parent
while current is not None:
nodes.append(current)
current = self.trace.nodes[current].parent
nodes.reverse()
return [self.hashes[n] for n in nodes] == hashes and self.trace.nodes[
nodes[0]
].tools == tools


class Replay:
def __init__(self, traces: list[Trace], *, mode: ReplayMode = "resume") -> None:
self.mode = mode
self._recordings: list[_Recording] = []
for trace in traces:
try:
self._recordings.append(_Recording(trace))
except Exception:
logger.warning(
"skipping a recording that does not load: id=%s",
trace.id,
exc_info=True,
)
self._bound: dict[str, _Recording] = {}

def claim(self, agent: str, task_key: str, trace_id: str) -> str:
"""Bind a newly minted trace of `agent` on `task_key` to a free recording and
return the id it runs under: the recording's own id the first time the recording
is claimed, else (a retry after a failed attempt) the trace's own."""
bound = {id(recording) for recording in self._bound.values()}
free = [
recording
for recording in self._recordings
if id(recording) not in bound
and recording.trace.agent.name == agent
and (self.mode == "playback" or recording.trace.task.key == task_key)
]
if not free:
return trace_id
recording = min(free, key=lambda recording: recording.adopted)
recording.used.clear()
if not recording.adopted:
recording.adopted = True
trace_id = recording.trace.id
self._bound[trace_id] = recording
return trace_id

def release(self, trace_id: str) -> None:
"""Free a failed trace's recording for the attempt that retries it."""
self._bound.pop(trace_id, None)

def plays_back(self, trace_id: str) -> bool:
"""Whether this trace plays its recording back, never sampling live."""
return self.mode == "playback" and trace_id in self._bound

def take(
self, trace_id: str, prompt: list[Message], tools: list[Tool] | None
) -> tuple[Response, ReplayMatch] | None:
"""The recorded reply for `prompt` and how it was matched, or None to sample live."""
recording = self._bound.get(trace_id)
if not prompt or recording is None:
return None
last = next(
(m for m in reversed(prompt) if isinstance(m, AssistantMessage)), None
)
key = None if last is None else graph.message_hash(last)
candidates = [
reply
for reply in recording.replies.get(key, [])
if reply not in recording.used
]
hashes = [graph.message_hash(message) for message in prompt]
exact = [
r for r in candidates if recording.is_prompt(r, hashes, list(tools or []))
]
if exact:
reply, match = exact[0], "exact"
elif self.mode == "playback" and candidates:
reply, match = candidates[0], "forced"
else:
return None
recording.used.add(reply)
source = recording.trace
call = next((c for c in source.calls if c.node == reply), None)
response = Response(
id=f"replay-{reply}",
created=0,
model=call.model if call and call.model else "",
message=source.nodes[reply].message.model_copy(deep=True),
finish_reason=call.finish_reason if call else None,
usage=call.usage if call else None,
)
return response, match
Loading
Loading