diff --git a/docs/v1/evaluation.md b/docs/v1/evaluation.md index f1155759aa..a94141c9a3 100644 --- a/docs/v1/evaluation.md +++ b/docs/v1/evaluation.md @@ -53,7 +53,21 @@ The output from evaluations are written into `outputs/----/ ## Resuming evaluations -`--resume ` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. +`--resume ` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. Rollouts the interruption cut off run again from the start; prime-rl's `uv run eval --resume` replays them from their live traces instead. + +## Replaying traces + +`vf.Replay` answers model calls with the replies in recorded traces. The harness runs every replayed tool call again, so it rebuilds the recorded rollout's files, processes, and tool state. It has two modes: + +- `resume` (default): replay while each prompt equals its recorded prompt, then sample live. This continues an interrupted rollout. +- `playback`: replay every recorded reply whatever the prompt, never sample live, and stop with `replay_ended` when the recording runs out. This reproduces a trace's deliverables, or drives one seat of a multi-agent env. + +```python +async with vf.Agent(config, replay=vf.Replay(traces)) as agent: + trace = await agent.run(task) +``` + +An env plays back one seat from `setup()`: `agents.solver.ctx = dataclasses.replace(agents.solver.ctx, replay=vf.Replay(traces, mode="playback"))`. `ModelCall.replayed` tags each replayed call: `exact` when its prompt equals the recorded prompt, `forced` when a playback serves it to a prompt that differs. Tool calls run again, side effects included. Replay is for evaluation only: the train client needs token ids. ## Disabling tools diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 78120a9f41..c758922030 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -76,6 +76,7 @@ Toolset, ToolsetConfig, ) +from verifiers.v1.replay import Replay from verifiers.v1.runtimes import ( ApptainerConfig, DockerConfig, @@ -229,6 +230,7 @@ "Trace", "TraceTask", "WireTrace", + "Replay", "Reward", "EnvInfo", "Episode", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 6d8e063c19..cbfc12925e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -26,6 +26,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer +from verifiers.v1.replay import Replay from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, @@ -268,6 +269,8 @@ def __init__( config: AgentConfig, *, interception: Interception | None = None, + name: str = "agent", + replay: Replay | None = None, ) -> None: from verifiers.v1.utils.loaders import harness_config_type, load_harness @@ -286,10 +289,12 @@ def __init__( config = config.model_copy(update={"sampling": Sampling()}) self.config = config self.harness = load_harness(config.harness) + self._name = name self.ctx = ModelContext( model=config.model, client=config.client or EvalClientConfig(), sampling=config.sampling, + replay=replay, ) self._closed = False self.runtime_config: RuntimeConfig = config.runtime @@ -562,6 +567,7 @@ def _rollout_params( timeouts = resolve_rollout_timeouts(self.timeout, task) return { "agent_config": self.config, + "agent_name": self._name, "harness": harness, "ctx": self.ctx, "runtime_config": runtime_config, @@ -612,11 +618,11 @@ def __init__( on_trace: Callable[[Trace], None] | None, on_discard: Callable[[Trace], None] | None, warned_resources: set, + replay: Replay | None = None, ) -> None: - super().__init__(config, interception=interception) + super().__init__(config, interception=interception, name=name, replay=replay) # Resource warnings dedupe env-wide, not per episode. self._warned_resources = warned_resources - self._name = name self._shared_tools = shared_tools self._task_cls = task_cls self._gate = gate diff --git a/verifiers/v1/clients/client.py b/verifiers/v1/clients/client.py index 5b4bab1a0b..6133d05394 100644 --- a/verifiers/v1/clients/client.py +++ b/verifiers/v1/clients/client.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from dataclasses import dataclass, field +from typing import TYPE_CHECKING from verifiers.v1.configs.client import ( BaseClientConfig, @@ -13,6 +14,9 @@ from verifiers.v1.graph import PendingTurn from verifiers.v1.types import Response, Sampling, SamplingConfig +if TYPE_CHECKING: + from verifiers.v1.replay import Replay + SESSION_ID_HEADER = "X-Session-ID" """Per-rollout routing header (the trace id, same value every turn), so a session-affinity router pins a rollout's turns to one engine and its growing prefix stays KV-cached.""" @@ -87,3 +91,5 @@ class ModelContext: model: str client: ClientConfig sampling: Sampling = field(default_factory=Sampling) + replay: "Replay | None" = field(default=None, compare=False) + """Recorded traces whose replies answer matching requests before the live model.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 69f97d7cb5..256fcd504a 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -226,6 +226,7 @@ def make(name: str, spec: AgentConfig) -> Agent: on_trace=on_trace, on_discard=on_discard, warned_resources=self._warned_resources, + replay=ctx.replay, ) agents = Agents(self.config, make) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d59118f877..7ea4d6a7c1 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -66,7 +66,7 @@ from verifiers.v1.semantic import ACPInfo, extract_acp_info from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan -from verifiers.v1.types import FinishReason, Response, Usage +from verifiers.v1.types import FinishReason, Request, Response, Usage logger = logging.getLogger(__name__) @@ -93,6 +93,29 @@ IDEMPOTENCY_CACHE_MAX_COMPLETED = 64 +def _replay_reply( + session: RolloutSession, dialect: Dialect, request: Request +) -> tuple[Response, Literal["exact", "forced"]] | None: + """The recorded reply that answers this request, rendered for the harness; None when + there is none, or when the lookup fails (logged).""" + if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig): + return None + try: + replayed = session.ctx.replay.take( + session.trace.id, request.messages, request.tools + ) + if replayed is not None: + replayed[0].raw = dialect.response_to_wire(replayed[0], session.ctx.model) + return replayed + except Exception: + logger.warning( + "replay lookup failed, sampling live: id=%s", + session.trace.id, + exc_info=True, + ) + return None + + def is_retried_request(headers: Mapping[str, str]) -> bool: try: return int(headers.get(RETRY_COUNT_HEADER, 0)) > 0 @@ -528,6 +551,7 @@ def record_call( error: BaseException | None = None, policy_paths: list[str] | None = None, acp: ACPInfo | None = None, + replayed: Literal["exact", "forced"] | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): the model + effective settings that went upstream, timing, and — when the call @@ -576,6 +600,7 @@ def record_call( if policy_paths else None, acp=acp, + replayed=replayed, ) ) session.trace.notify() @@ -777,11 +802,25 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + replayed = _replay_reply(session, dialect, model_request) + if ( + replayed is None + and session.ctx.replay is not None + and session.ctx.replay.plays_back(session.trace.id) + ): + # A playback never samples live: its recording ended here. + turn.commit_prompt() + session.trace.stop("replay_ended") + return web.json_response( + dialect.error_body("rollout stopped: replay_ended"), status=400 + ) try: try: # What actually goes upstream: the native body with the rollout's model + # sampling imposed — recorded raw on the trace, per call. - if relay: + if replayed is not None: + call_response = replayed[0] + elif relay: reply = await session.client.relay( dialect, body, @@ -809,7 +848,9 @@ async def sample() -> web.Response: ) response_rewrites = [] stopped = None - if session.response_interceptors or session.response_stops: + if replayed is None and ( + session.response_interceptors or session.response_stops + ): ( call_response, response_rewrites, @@ -891,6 +932,7 @@ async def sample() -> web.Response: error=error, policy_paths=policy_paths, acp=acp, + replayed=replayed[1] if replayed is not None else None, ) return serve(call_response, events) diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py new file mode 100644 index 0000000000..849670bb9e --- /dev/null +++ b/verifiers/v1/replay.py @@ -0,0 +1,151 @@ +"""Replay recorded model replies, in one of two modes. + +`resume` (the default) continues an interrupted rollout: it replays recorded replies while +each prompt is exactly its recorded prompt (`exact`), then samples live from the first +prompt that differs. + +`playback` plays a recording back, on any task: it replays every recorded reply whatever the +tools returned or other agents said, never samples live, and stops the rollout +(`replay_ended`) when the recording runs out. A reply played onto a prompt that differs from +its recorded prompt is `forced`: the recorded model never saw that prompt. + +A rollout whose context carries a `Replay` claims a free recording of its agent (on its task, +in resume) and runs under the recording's trace id. A failed attempt frees its recording, so +the attempt that retries it replays the recording again, under its own id. Each model +request is answered with the recorded reply that followed the prompt's last assistant +message, or that opened a branch when the prompt has none. The harness still runs every tool +call, so its runtime and program state are rebuilt.""" + +import logging +from dataclasses import dataclass, field +from typing import Literal + +from verifiers.v1 import graph +from verifiers.v1.trace import Trace +from verifiers.v1.types import AssistantMessage, Message, Response, Tool + +logger = logging.getLogger(__name__) + +ReplayMode = Literal["resume", "playback"] +ReplayMatch = Literal["exact", "forced"] + + +@dataclass(eq=False) +class _Recording: + trace: Trace + adopted: bool = False + used: set[int] = field(default_factory=set) + hashes: list[str] = field(default_factory=list) + replies: dict[str | None, list[int]] = field(default_factory=dict) + """Recorded replies by the hash of the assistant message their prompt ends after.""" + + def __post_init__(self) -> None: + nodes = self.trace.nodes + self.hashes = [graph.message_hash(node.message) for node in nodes] + for reply, node in enumerate(nodes): + if not node.sampled: + continue + last = node.parent + while last is not None and not isinstance( + nodes[last].message, AssistantMessage + ): + last = nodes[last].parent + key = None if last is None else self.hashes[last] + self.replies.setdefault(key, []).append(reply) + + def is_prompt(self, reply: int, hashes: list[str], tools: list[Tool]) -> bool: + """Whether a prompt with these message hashes and tools is `reply`'s recorded prompt.""" + nodes: list[int] = [] + current = self.trace.nodes[reply].parent + while current is not None: + nodes.append(current) + current = self.trace.nodes[current].parent + nodes.reverse() + return [self.hashes[n] for n in nodes] == hashes and self.trace.nodes[ + nodes[0] + ].tools == tools + + +class Replay: + def __init__(self, traces: list[Trace], *, mode: ReplayMode = "resume") -> None: + self.mode = mode + self._recordings: list[_Recording] = [] + for trace in traces: + try: + self._recordings.append(_Recording(trace)) + except Exception: + logger.warning( + "skipping a recording that does not load: id=%s", + trace.id, + exc_info=True, + ) + self._bound: dict[str, _Recording] = {} + + def claim(self, agent: str, task_key: str, trace_id: str) -> str: + """Bind a newly minted trace of `agent` on `task_key` to a free recording and + return the id it runs under: the recording's own id the first time the recording + is claimed, else (a retry after a failed attempt) the trace's own.""" + bound = {id(recording) for recording in self._bound.values()} + free = [ + recording + for recording in self._recordings + if id(recording) not in bound + and recording.trace.agent.name == agent + and (self.mode == "playback" or recording.trace.task.key == task_key) + ] + if not free: + return trace_id + recording = min(free, key=lambda recording: recording.adopted) + recording.used.clear() + if not recording.adopted: + recording.adopted = True + trace_id = recording.trace.id + self._bound[trace_id] = recording + return trace_id + + def release(self, trace_id: str) -> None: + """Free a failed trace's recording for the attempt that retries it.""" + self._bound.pop(trace_id, None) + + def plays_back(self, trace_id: str) -> bool: + """Whether this trace plays its recording back, never sampling live.""" + return self.mode == "playback" and trace_id in self._bound + + def take( + self, trace_id: str, prompt: list[Message], tools: list[Tool] | None + ) -> tuple[Response, ReplayMatch] | None: + """The recorded reply for `prompt` and how it was matched, or None to sample live.""" + recording = self._bound.get(trace_id) + if not prompt or recording is None: + return None + last = next( + (m for m in reversed(prompt) if isinstance(m, AssistantMessage)), None + ) + key = None if last is None else graph.message_hash(last) + candidates = [ + reply + for reply in recording.replies.get(key, []) + if reply not in recording.used + ] + hashes = [graph.message_hash(message) for message in prompt] + exact = [ + r for r in candidates if recording.is_prompt(r, hashes, list(tools or [])) + ] + if exact: + reply, match = exact[0], "exact" + elif self.mode == "playback" and candidates: + reply, match = candidates[0], "forced" + else: + return None + recording.used.add(reply) + source = recording.trace + call = next((c for c in source.calls if c.node == reply), None) + response = Response( + id=f"replay-{reply}", + created=0, + model=call.model if call and call.model else "", + message=source.nodes[reply].message.model_copy(deep=True), + finish_reason=call.finish_reason if call else None, + usage=call.usage if call else None, + ) + return response, match diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 5e6d5bbb92..57fed12798 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -94,6 +94,7 @@ def __init__( *, task: Task, agent_config: AgentConfig, + agent_name: str = "agent", harness: Harness, ctx: ModelContext, runtime_config: RuntimeConfig, @@ -128,8 +129,10 @@ def __init__( state=state_cls(type(task))(), # The seat's resolved config, role overrides included — the agent # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), + agent=AgentInfo(config=agent_config, name=agent_name), ) + if ctx.replay is not None: + self.trace.id = ctx.replay.claim(agent_name, task.key, self.trace.id) if on_trace is not None: on_trace(self.trace) interceptors = [ @@ -568,6 +571,8 @@ async def close(self) -> Trace: await self._stack.aclose() trace.is_completed = True trace.ok = not self._failed + if not trace.ok and self.ctx.replay is not None: + self.ctx.replay.release(trace.id) now = time.time() for span in ( trace.timing.boot, diff --git a/verifiers/v1/serve/client.py b/verifiers/v1/serve/client.py index 2d72516a60..99f3757c12 100644 --- a/verifiers/v1/serve/client.py +++ b/verifiers/v1/serve/client.py @@ -190,13 +190,15 @@ async def run( sampling: SamplingConfig, task_data: dict, on_delta: Callable[[dict], None] | None = None, + replay: list[dict] | None = None, ) -> WireEpisode: """Run one rollout; return its episode record — flat traces (typed `Trace[WireTaskData]`) plus the shared stamp. The server takes the task itself (`task_data`, its dumped `TaskData`). The traces stream in as the rollout runs, one delta per turn or phase change (`serve.delta`); `on_delta` sees each as it lands, so a caller can relay or persist the stream. The - episode returned is assembled from the same deltas.""" + episode returned is assembled from the same deltas. `replay` carries the dumped + traces of an interrupted attempt of this episode (`verifiers.v1.replay`).""" assembly = EpisodeAssembly() def apply(data: bytes) -> None: @@ -211,6 +213,7 @@ def apply(data: bytes) -> None: client=client, model=model, sampling=sampling, + replay=replay or [], ), RunResponse, on_delta=apply, diff --git a/verifiers/v1/serve/server.py b/verifiers/v1/serve/server.py index 8e04bc8069..1bb65f525b 100644 --- a/verifiers/v1/serve/server.py +++ b/verifiers/v1/serve/server.py @@ -1,5 +1,6 @@ import asyncio import logging +from dataclasses import replace import msgpack import zmq @@ -8,6 +9,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.client import ClientConfig from verifiers.v1.configs.env import EnvConfig +from verifiers.v1.replay import Replay from verifiers.v1.serve.delta import DeltaStreamer, TraceSummary, dump, pack from verifiers.v1.serve.encoding import msgpack_encoder from verifiers.v1.serve.types import ( @@ -19,6 +21,7 @@ RunResponse, ) from verifiers.v1.task import Task +from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig from verifiers.v1.utils.loaders import load_environment @@ -98,6 +101,16 @@ async def _run( self, req: RunRequest, client_id: bytes, request_id: bytes ) -> RunResponse: ctx = self._context(req.client, req.model, req.sampling) + if req.replay: + traces = [] + for trace in req.replay: + try: + traces.append(WireTrace.model_validate(trace)) + except ValueError: + logger.warning( + "skipping a recording that does not validate", exc_info=True + ) + ctx = replace(ctx, replay=Replay(traces)) (slot,) = self.env.slots(self._build_task(req.task_data)) async def send_delta(delta: dict) -> None: diff --git a/verifiers/v1/serve/types.py b/verifiers/v1/serve/types.py index 1ea41b04cd..7c1b44f207 100644 --- a/verifiers/v1/serve/types.py +++ b/verifiers/v1/serve/types.py @@ -48,6 +48,8 @@ class RunRequest(BaseRequest): client: ClientConfig model: str sampling: SamplingConfig + replay: list[dict] = Field(default_factory=list) + """Dumped traces of an interrupted attempt of this episode, replayed before sampling live.""" class RunResponse(BaseResponse): diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index c3b4f6c58d..e84e1184e8 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -5,7 +5,7 @@ import traceback import uuid from collections.abc import Callable, Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Generic +from typing import TYPE_CHECKING, Any, Generic, Literal import numpy as np from pydantic import BaseModel, Field, PrivateAttr, computed_field, field_serializer @@ -173,6 +173,10 @@ class ModelCall(BaseModel): """Policy mediation applied to the request before this call.""" acp: ACPInfo | None = None """Metadata advertised by the ACP harness for this model request.""" + replayed: Literal["exact", "forced"] | None = None + """Answered from a recorded trace (`ModelContext.replay`) instead of the provider: the + prompt was exactly its recorded prompt, or a playback forced the reply onto a prompt that + differs from it.""" def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall, int]]: