From fe676159fed6ebaee2aea3931c8e14214a219320 Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Fri, 2 Oct 2026 21:20:41 +0000 Subject: [PATCH 1/8] Replay recorded model replies to resume interrupted rollouts A rollout given recorded traces (ModelContext.replay, RunRequest.replay, or Agent(replay=...)) continues under the recorded trace id. The interception server answers each model request whose prompt matches the recording with the recorded reply, before any upstream call, and the harness runs the tool calls again, which rebuilds the runtime and program state. A prompt matches its exact recorded path or, failing that, the path with the same structure. The first unmatched request goes to the live model. ModelCall.replayed records how each answered call matched. Each dialect can now render a vf Response as its native response, and the Anthropic and Responses dialects stream responses with several blocks or items. --- verifiers/v1/agent.py | 10 ++- verifiers/v1/clients/client.py | 6 ++ verifiers/v1/dialects/anthropic.py | 103 ++++++++++++++++----- verifiers/v1/dialects/base.py | 4 + verifiers/v1/dialects/chat.py | 30 +++++++ verifiers/v1/dialects/responses.py | 133 +++++++++++++++++++++------ verifiers/v1/env.py | 1 + verifiers/v1/interception/server.py | 37 +++++++- verifiers/v1/replay.py | 135 ++++++++++++++++++++++++++++ verifiers/v1/rollout.py | 7 +- verifiers/v1/serve/client.py | 5 +- verifiers/v1/serve/server.py | 6 ++ verifiers/v1/serve/types.py | 2 + verifiers/v1/trace.py | 5 +- 14 files changed, 425 insertions(+), 59 deletions(-) create mode 100644 verifiers/v1/replay.py diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 6d8e063c19..cbfc12925e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -26,6 +26,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer +from verifiers.v1.replay import Replay from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, @@ -268,6 +269,8 @@ def __init__( config: AgentConfig, *, interception: Interception | None = None, + name: str = "agent", + replay: Replay | None = None, ) -> None: from verifiers.v1.utils.loaders import harness_config_type, load_harness @@ -286,10 +289,12 @@ def __init__( config = config.model_copy(update={"sampling": Sampling()}) self.config = config self.harness = load_harness(config.harness) + self._name = name self.ctx = ModelContext( model=config.model, client=config.client or EvalClientConfig(), sampling=config.sampling, + replay=replay, ) self._closed = False self.runtime_config: RuntimeConfig = config.runtime @@ -562,6 +567,7 @@ def _rollout_params( timeouts = resolve_rollout_timeouts(self.timeout, task) return { "agent_config": self.config, + "agent_name": self._name, "harness": harness, "ctx": self.ctx, "runtime_config": runtime_config, @@ -612,11 +618,11 @@ def __init__( on_trace: Callable[[Trace], None] | None, on_discard: Callable[[Trace], None] | None, warned_resources: set, + replay: Replay | None = None, ) -> None: - super().__init__(config, interception=interception) + super().__init__(config, interception=interception, name=name, replay=replay) # Resource warnings dedupe env-wide, not per episode. self._warned_resources = warned_resources - self._name = name self._shared_tools = shared_tools self._task_cls = task_cls self._gate = gate diff --git a/verifiers/v1/clients/client.py b/verifiers/v1/clients/client.py index 5b4bab1a0b..6133d05394 100644 --- a/verifiers/v1/clients/client.py +++ b/verifiers/v1/clients/client.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from dataclasses import dataclass, field +from typing import TYPE_CHECKING from verifiers.v1.configs.client import ( BaseClientConfig, @@ -13,6 +14,9 @@ from verifiers.v1.graph import PendingTurn from verifiers.v1.types import Response, Sampling, SamplingConfig +if TYPE_CHECKING: + from verifiers.v1.replay import Replay + SESSION_ID_HEADER = "X-Session-ID" """Per-rollout routing header (the trace id, same value every turn), so a session-affinity router pins a rollout's turns to one engine and its growing prefix stays KV-cached.""" @@ -87,3 +91,5 @@ class ModelContext: model: str client: ClientConfig sampling: Sampling = field(default_factory=Sampling) + replay: "Replay | None" = field(default=None, compare=False) + """Recorded traces whose replies answer matching requests before the live model.""" diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 4c5983bef6..f95f586220 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -599,43 +599,98 @@ def rewrite_response(self, raw: dict, text: str) -> None: raw["stop_reason"] = "end_turn" raw["stop_sequence"] = None + def response_to_wire(self, response: Response, model: str) -> dict: + message = response.message + content = [dict(block) for block in message.provider_state or []] + if message.content: + content.append({"type": "text", "text": message.content}) + for call in message.tool_calls or []: + block = { + "type": "tool_use", + "id": call.id, + "name": call.name, + "input": json.loads(call.arguments or "{}"), + } + if call.namespace: + block["toolset_name"] = call.namespace + content.append(block) + stop = {"length": "max_tokens", "tool_calls": "tool_use"}.get( + response.finish_reason or "", "end_turn" + ) + raw: dict = { + "id": response.id, + "type": "message", + "role": "assistant", + "model": model, + "content": content, + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + if usage := response.usage: + raw["usage"] = { + "input_tokens": usage.prompt_tokens, + "output_tokens": usage.completion_tokens, + "cache_read_input_tokens": usage.cached_input_tokens, + } + if usage.reasoning_tokens is not None: + raw["usage"]["output_tokens_details"] = { + "thinking_tokens": usage.reasoning_tokens + } + return raw + def stream_events(self, raw: dict) -> list[bytes]: def event(kind: str, payload: dict) -> bytes: - return f"event: {kind}\ndata: {json.dumps(payload)}\n\n".encode() + return f"event: {kind}\ndata: {json.dumps({'type': kind, **payload})}\n\n".encode() - text = raw["content"][0]["text"] head = {**raw, "content": [], "stop_reason": None, "stop_sequence": None} if isinstance(usage := head.get("usage"), dict): head["usage"] = {**usage, "output_tokens": 0} - return [ - event("message_start", {"type": "message_start", "message": head}), - event( - "content_block_start", - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, - ), - event( - "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": text}, - }, - ), - event("content_block_stop", {"type": "content_block_stop", "index": 0}), + events = [event("message_start", {"message": head})] + for index, block in enumerate(raw["content"]): + kind = block["type"] + if kind == "text": + start = {**block, "text": ""} + deltas = [{"type": "text_delta", "text": block["text"]}] + elif kind == "thinking": + start = {**block, "thinking": "", "signature": ""} + deltas = [{"type": "thinking_delta", "thinking": block["thinking"]}] + if block.get("signature"): + deltas.append( + {"type": "signature_delta", "signature": block["signature"]} + ) + elif kind == "tool_use": + start = {**block, "input": {}} + deltas = [ + { + "type": "input_json_delta", + "partial_json": json.dumps(block["input"]), + } + ] + else: + start, deltas = block, [] + events.append( + event("content_block_start", {"index": index, "content_block": start}) + ) + events += [ + event("content_block_delta", {"index": index, "delta": delta}) + for delta in deltas + ] + events.append(event("content_block_stop", {"index": index})) + events += [ event( "message_delta", { - "type": "message_delta", - "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "delta": { + "stop_reason": raw.get("stop_reason") or "end_turn", + "stop_sequence": raw.get("stop_sequence"), + }, "usage": raw.get("usage") or {}, }, ), - event("message_stop", {"type": "message_stop"}), + event("message_stop", {}), ] + return events def stream_parser(self) -> StreamParser: return AnthropicStreamParser(self.validate_response) diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index d68350f328..6aae355cf4 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -350,6 +350,10 @@ def parse_sampling(self, body: RawRequest) -> Sampling: def parse_response(self, response: RespT) -> Response: """A native (non-streamed) response -> the vf `Response` we consume.""" + @abstractmethod + def response_to_wire(self, response: Response, model: str) -> dict: + """A vf `Response` -> the native response that parses back to it.""" + def validate_response(self, raw: dict) -> RespT: """Validate a native response, normalizing provider-compatible extensions if needed.""" return self.response_type.model_validate(raw) diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index e025b97f81..af7c222ad8 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -537,6 +537,36 @@ def parse_sampling(self, body: RawRequest) -> Sampling: def parse_response(self, response: ChatCompletion) -> Response: return response_from_wire(response) + def response_to_wire(self, response: Response, model: str) -> dict: + message = message_to_wire(response.message) + if response.message.reasoning_content is not None: + message["reasoning_content"] = response.message.reasoning_content + finish = response.finish_reason or ( + "tool_calls" if response.message.tool_calls else "stop" + ) + raw: dict = { + "id": response.id, + "object": "chat.completion", + "created": response.created, + "model": model, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + } + if usage := response.usage: + raw["usage"] = { + "prompt_tokens": usage.input_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + } + if usage.cached_input_tokens is not None: + raw["usage"]["prompt_tokens_details"] = { + "cached_tokens": usage.cached_input_tokens + } + if usage.reasoning_tokens is not None: + raw["usage"]["completion_tokens_details"] = { + "reasoning_tokens": usage.reasoning_tokens + } + return raw + def rewrite_request(self, body: dict, before: Request, after: Request) -> None: for native, original, rewritten in zip( body.get("messages", []), before.messages, after.messages, strict=True diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index e8876096b3..cf93d65ff4 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -704,40 +704,119 @@ def stream_error(self, error: dict) -> bytes: } return f"event: response.failed\ndata: {json.dumps(payload)}\n\n".encode() + def response_to_wire(self, response: Response, model: str) -> dict: + message = response.message + output = message.provider_state + if output is None: + output = [ + { + "type": "function_call" + if call.type == "function" + else "custom_tool_call", + "id": f"fc_{call.id}", + "call_id": call.id, + "name": call.name, + **({"namespace": call.namespace} if call.namespace else {}), + "arguments" if call.type == "function" else "input": call.arguments, + "status": "completed", + } + for call in message.tool_calls or [] + ] + if message.content: + output.insert( + 0, + { + "type": "message", + "id": f"msg_{response.id}", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": message.content, + "annotations": [], + } + ], + }, + ) + usage = response.usage + return { + "id": response.id, + "object": "response", + "created_at": response.created, + "model": model, + "status": "incomplete" + if response.finish_reason == "length" + else "completed", + "incomplete_details": {"reason": "max_output_tokens"} + if response.finish_reason == "length" + else None, + "error": None, + "output": output, + "usage": { + "input_tokens": usage.input_tokens, + "input_tokens_details": { + "cached_tokens": usage.cached_input_tokens or 0 + }, + "output_tokens": usage.completion_tokens, + "output_tokens_details": { + "reasoning_tokens": usage.reasoning_tokens or 0 + }, + "total_tokens": usage.total_tokens, + } + if usage + else None, + } + def stream_events(self, raw: dict) -> list[bytes]: - item = raw["output"][0] - part = item["content"][0] - common = {"output_index": 0, "item_id": item["id"], "content_index": 0} - logprobs = part.get("logprobs") or [] head = { **raw, "status": "in_progress", "output": [], "completed_at": None, } - events = [ - ("response.created", {"response": head}), - ( - "response.output_item.added", - {"output_index": 0, "item": {**item, "content": []}}, - ), - ( - "response.content_part.added", - {**common, "part": {**part, "text": ""}}, - ), - # `logprobs` is required on both text events; carry the part's own. - ( - "response.output_text.delta", - {**common, "delta": part["text"], "logprobs": logprobs}, - ), - ( - "response.output_text.done", - {**common, "text": part["text"], "logprobs": logprobs}, - ), - ("response.content_part.done", {**common, "part": part}), - ("response.output_item.done", {"output_index": 0, "item": item}), - ("response.completed", {"response": raw}), - ] + events: list[tuple[str, dict]] = [("response.created", {"response": head})] + for index, item in enumerate(raw["output"]): + parts = item.get("content") if item.get("type") == "message" else None + events.append( + ( + "response.output_item.added", + {"output_index": index, "item": {**item, "content": []}} + if parts is not None + else {"output_index": index, "item": item}, + ) + ) + for content_index, part in enumerate(parts or []): + common = { + "output_index": index, + "item_id": item["id"], + "content_index": content_index, + } + text = part.get("type") == "output_text" + events.append( + ( + "response.content_part.added", + {**common, "part": {**part, "text": ""} if text else part}, + ) + ) + if text: + # `logprobs` is required on both text events; carry the part's own. + logprobs = part.get("logprobs") or [] + events += [ + ( + "response.output_text.delta", + {**common, "delta": part["text"], "logprobs": logprobs}, + ), + ( + "response.output_text.done", + {**common, "text": part["text"], "logprobs": logprobs}, + ), + ] + events.append(("response.content_part.done", {**common, "part": part})) + events.append( + ("response.output_item.done", {"output_index": index, "item": item}) + ) + events.append(("response.completed", {"response": raw})) return [ *( f"data: {json.dumps({'type': kind, 'sequence_number': i, **data})}\n\n".encode() diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 69f97d7cb5..256fcd504a 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -226,6 +226,7 @@ def make(name: str, spec: AgentConfig) -> Agent: on_trace=on_trace, on_discard=on_discard, warned_resources=self._warned_resources, + replay=ctx.replay, ) agents = Agents(self.config, make) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d59118f877..94e043f92f 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -66,7 +66,7 @@ from verifiers.v1.semantic import ACPInfo, extract_acp_info from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan -from verifiers.v1.types import FinishReason, Response, Usage +from verifiers.v1.types import FinishReason, Request, Response, Usage logger = logging.getLogger(__name__) @@ -93,6 +93,29 @@ IDEMPOTENCY_CACHE_MAX_COMPLETED = 64 +def _replay_reply( + session: RolloutSession, dialect: Dialect, request: Request +) -> tuple[Response, Literal["exact", "structural"]] | None: + """The recorded reply that answers this request, rendered for the harness; None to + sample live. Replay only saves work, so a failure in it falls back to the model.""" + if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig): + return None + try: + replayed = session.ctx.replay.take( + session.trace.id, request.messages, request.tools + ) + if replayed is not None: + replayed[0].raw = dialect.response_to_wire(replayed[0], session.ctx.model) + return replayed + except Exception: + logger.warning( + "replay lookup failed, sampling live: id=%s", + session.trace.id, + exc_info=True, + ) + return None + + def is_retried_request(headers: Mapping[str, str]) -> bool: try: return int(headers.get(RETRY_COUNT_HEADER, 0)) > 0 @@ -528,6 +551,7 @@ def record_call( error: BaseException | None = None, policy_paths: list[str] | None = None, acp: ACPInfo | None = None, + replayed: Literal["exact", "structural"] | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): the model + effective settings that went upstream, timing, and — when the call @@ -576,6 +600,7 @@ def record_call( if policy_paths else None, acp=acp, + replayed=replayed, ) ) session.trace.notify() @@ -777,11 +802,14 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + replayed = _replay_reply(session, dialect, model_request) try: try: # What actually goes upstream: the native body with the rollout's model + # sampling imposed — recorded raw on the trace, per call. - if relay: + if replayed is not None: + call_response = replayed[0] + elif relay: reply = await session.client.relay( dialect, body, @@ -809,7 +837,9 @@ async def sample() -> web.Response: ) response_rewrites = [] stopped = None - if session.response_interceptors or session.response_stops: + if replayed is None and ( + session.response_interceptors or session.response_stops + ): ( call_response, response_rewrites, @@ -891,6 +921,7 @@ async def sample() -> web.Response: error=error, policy_paths=policy_paths, acp=acp, + replayed=replayed[1] if replayed is not None else None, ) return serve(call_response, events) diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py new file mode 100644 index 0000000000..a907c0a8ff --- /dev/null +++ b/verifiers/v1/replay.py @@ -0,0 +1,135 @@ +"""Resume interrupted rollouts by replaying their recorded model replies. + +A rollout whose context carries a `Replay` claims the first unclaimed recorded trace of +its agent on its task and continues under that trace's id. Each model request is matched to a path of +the recording and answered with the recorded reply; the harness still runs every tool call, +so its runtime and program state are rebuilt. A prompt matches its exact recorded path, or +else (`structural`) the path with the same tools and structure: every assistant message +identical, every other message in the same role (and for a tool result, the same call), its +text free to differ — a timestamp, a hostname, a session path. A request no path answers +goes to the live model, and so does everything after it, since its prompt carries the live +reply.""" + +from collections import Counter +from dataclasses import dataclass, field +from typing import Literal + +from verifiers.v1 import graph +from verifiers.v1.trace import Trace +from verifiers.v1.types import AssistantMessage, Message, Response, Tool, ToolMessage + +ReplayMatch = Literal["exact", "structural"] + + +@dataclass +class _Recording: + trace: Trace + children: dict[int | None, list[int]] = field(default_factory=dict) + hashes: list[str] = field(default_factory=list) + used: Counter[int] = field(default_factory=Counter) + + def __post_init__(self) -> None: + for node_id, node in enumerate(self.trace.nodes): + self.children.setdefault(node.parent, []).append(node_id) + self.hashes.append(graph.message_hash(node.message)) + + +class Replay: + def __init__(self, traces: list[Trace]) -> None: + self._unclaimed = list(traces) + self._claimed: dict[str, _Recording] = {} + + def claim(self, agent: str, task_key: str) -> str | None: + """The recorded trace id a newly minted trace of `agent` on `task_key` continues.""" + for trace in self._unclaimed: + if trace.agent.name == agent and trace.task.key == task_key: + self._unclaimed.remove(trace) + self._claimed[trace.id] = _Recording(trace) + return trace.id + return None + + def take( + self, trace_id: str, prompt: list[Message], tools: list[Tool] | None + ) -> tuple[Response, ReplayMatch] | None: + """The recorded reply to `prompt` and how the prompt matched, or None to sample live.""" + recording = self._claimed.get(trace_id) + if not prompt or recording is None: + return None + source = recording.trace + turn = graph.prepare_turn(source, prompt, tools) + match: ReplayMatch = "exact" + if len(turn.prefix_node_ids) == len(prompt): + parent = turn.prefix_node_ids[-1] + elif (parent := _structural_match(recording, prompt, tools)) is not None: + match = "structural" + else: + return None + replies = [ + n for n in recording.children.get(parent, []) if source.nodes[n].sampled + ] + if recording.used[parent] >= len(replies): + return None + node = replies[recording.used[parent]] + recording.used[parent] += 1 + call = next((c for c in source.calls if c.node == node), None) + response = Response( + id=f"replay-{node}", + created=0, + model=call.model if call and call.model else "", + message=source.nodes[node].message.model_copy(deep=True), + finish_reason=call.finish_reason if call else None, + usage=call.usage if call else None, + ) + return response, match + + +def _structural_match( + recording: _Recording, prompt: list[Message], tools: list[Tool] | None +) -> int | None: + source = recording.trace + parent: int | None = None + for message in prompt: + options = recording.children.get(parent, []) + if parent is None: + options = [n for n in options if source.nodes[n].tools == list(tools or [])] + key = graph.message_hash(message) + exact = [n for n in options if recording.hashes[n] == key] + if exact: + parent = exact[0] + continue + if isinstance(message, AssistantMessage): + return None + alike = [ + n + for n in options + if source.nodes[n].message.role == message.role + and ( + not isinstance(message, ToolMessage) + or source.nodes[n].message.tool_call_id == message.tool_call_id + ) + ] + if not alike: + return None + text = _text(message) + parent = max( + alike, key=lambda n: _overlap(_text(source.nodes[n].message), text) + ) + return parent + + +def _text(message: Message) -> str: + if isinstance(message.content, str): + return message.content + return "".join(getattr(part, "text", "") for part in message.content or []) + + +def _overlap(a: str, b: str) -> int: + """Shared leading plus trailing characters: how alike two variants of one message are.""" + head = next( + (i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b)) + ) + tail = next( + (i for i, (x, y) in enumerate(zip(reversed(a), reversed(b))) if x != y), + min(len(a), len(b)), + ) + return head + tail diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 0e3613e1a3..ae5138fa53 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -60,6 +60,7 @@ def __init__( *, task: Task, agent_config: AgentConfig, + agent_name: str = "agent", harness: Harness, ctx: ModelContext, runtime_config: RuntimeConfig, @@ -94,8 +95,12 @@ def __init__( state=state_cls(type(task))(), # The seat's resolved config, role overrides included — the agent # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), + agent=AgentInfo(config=agent_config, name=agent_name), ) + if ctx.replay is not None and ( + claimed := ctx.replay.claim(agent_name, task.key) + ): + self.trace.id = claimed if on_trace is not None: on_trace(self.trace) interceptors = [ diff --git a/verifiers/v1/serve/client.py b/verifiers/v1/serve/client.py index 2d72516a60..99f3757c12 100644 --- a/verifiers/v1/serve/client.py +++ b/verifiers/v1/serve/client.py @@ -190,13 +190,15 @@ async def run( sampling: SamplingConfig, task_data: dict, on_delta: Callable[[dict], None] | None = None, + replay: list[dict] | None = None, ) -> WireEpisode: """Run one rollout; return its episode record — flat traces (typed `Trace[WireTaskData]`) plus the shared stamp. The server takes the task itself (`task_data`, its dumped `TaskData`). The traces stream in as the rollout runs, one delta per turn or phase change (`serve.delta`); `on_delta` sees each as it lands, so a caller can relay or persist the stream. The - episode returned is assembled from the same deltas.""" + episode returned is assembled from the same deltas. `replay` carries the dumped + traces of an interrupted attempt of this episode (`verifiers.v1.replay`).""" assembly = EpisodeAssembly() def apply(data: bytes) -> None: @@ -211,6 +213,7 @@ def apply(data: bytes) -> None: client=client, model=model, sampling=sampling, + replay=replay or [], ), RunResponse, on_delta=apply, diff --git a/verifiers/v1/serve/server.py b/verifiers/v1/serve/server.py index 8e04bc8069..e89aa6783f 100644 --- a/verifiers/v1/serve/server.py +++ b/verifiers/v1/serve/server.py @@ -1,5 +1,6 @@ import asyncio import logging +from dataclasses import replace import msgpack import zmq @@ -8,6 +9,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.client import ClientConfig from verifiers.v1.configs.env import EnvConfig +from verifiers.v1.replay import Replay from verifiers.v1.serve.delta import DeltaStreamer, TraceSummary, dump, pack from verifiers.v1.serve.encoding import msgpack_encoder from verifiers.v1.serve.types import ( @@ -19,6 +21,7 @@ RunResponse, ) from verifiers.v1.task import Task +from verifiers.v1.trace import WireTrace from verifiers.v1.types import SamplingConfig from verifiers.v1.utils.loaders import load_environment @@ -98,6 +101,9 @@ async def _run( self, req: RunRequest, client_id: bytes, request_id: bytes ) -> RunResponse: ctx = self._context(req.client, req.model, req.sampling) + if req.replay: + traces = [WireTrace.model_validate(trace) for trace in req.replay] + ctx = replace(ctx, replay=Replay(traces)) (slot,) = self.env.slots(self._build_task(req.task_data)) async def send_delta(delta: dict) -> None: diff --git a/verifiers/v1/serve/types.py b/verifiers/v1/serve/types.py index 1ea41b04cd..7c1b44f207 100644 --- a/verifiers/v1/serve/types.py +++ b/verifiers/v1/serve/types.py @@ -48,6 +48,8 @@ class RunRequest(BaseRequest): client: ClientConfig model: str sampling: SamplingConfig + replay: list[dict] = Field(default_factory=list) + """Dumped traces of an interrupted attempt of this episode, replayed before sampling live.""" class RunResponse(BaseResponse): diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index be14b1d425..a8ffe75b67 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -5,7 +5,7 @@ import traceback import uuid from collections.abc import Callable, Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Generic +from typing import TYPE_CHECKING, Any, Generic, Literal import numpy as np from pydantic import BaseModel, Field, PrivateAttr, computed_field, field_serializer @@ -173,6 +173,9 @@ class ModelCall(BaseModel): """Policy mediation applied to the request before this call.""" acp: ACPInfo | None = None """Metadata advertised by the ACP harness for this model request.""" + replayed: Literal["exact", "structural"] | None = None + """Answered from a recorded trace (`ModelContext.replay`) instead of the provider, and + whether the prompt matched its recorded path exactly or by structure.""" def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall, int]]: From eea1b3ba7623e9438ef6b0523ded6bee98f30bdf Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Fri, 2 Oct 2026 21:40:12 +0000 Subject: [PATCH 2/8] Match replayed prompts by volatile-only differences and replay retries Structural matching now requires every non-assistant message to differ from its recording only in volatile runs (uuids, hashes, hostnames, durations, clock times, epoch timestamps); any other difference samples live. Every branch that fits the prompt stays a candidate until a message tells siblings apart, so parallel subagents with near-identical prompts resolve correctly. Tools compare order-insensitively with volatile runs masked. A failed attempt frees its recording, so an agent or episode retry replays it again under a fresh trace id. A lookup or rendering failure falls back to the live model. Replay is exported from verifiers.v1. --- verifiers/v1/__init__.py | 2 + verifiers/v1/replay.py | 193 ++++++++++++++++++++++++++------------- verifiers/v1/rollout.py | 8 +- 3 files changed, 137 insertions(+), 66 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 78120a9f41..c758922030 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -76,6 +76,7 @@ Toolset, ToolsetConfig, ) +from verifiers.v1.replay import Replay from verifiers.v1.runtimes import ( ApptainerConfig, DockerConfig, @@ -229,6 +230,7 @@ "Trace", "TraceTask", "WireTrace", + "Replay", "Reward", "EnvInfo", "Episode", diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py index a907c0a8ff..9f3f9fd705 100644 --- a/verifiers/v1/replay.py +++ b/verifiers/v1/replay.py @@ -1,15 +1,20 @@ """Resume interrupted rollouts by replaying their recorded model replies. -A rollout whose context carries a `Replay` claims the first unclaimed recorded trace of -its agent on its task and continues under that trace's id. Each model request is matched to a path of -the recording and answered with the recorded reply; the harness still runs every tool call, -so its runtime and program state are rebuilt. A prompt matches its exact recorded path, or -else (`structural`) the path with the same tools and structure: every assistant message -identical, every other message in the same role (and for a tool result, the same call), its -text free to differ — a timestamp, a hostname, a session path. A request no path answers -goes to the live model, and so does everything after it, since its prompt carries the live -reply.""" - +A rollout whose context carries a `Replay` claims a free recorded trace of its agent on +its task and continues under that trace's id. A failed attempt frees its recording, so +the attempt that retries it replays the recording again, under its own id. Each model +request is matched to a path of the recording and answered with the recorded reply; the +harness still runs every tool call, so its runtime and program state are rebuilt. + +A prompt matches its exact recorded path, or else (`structural`) a path with the same +tools and structure: every assistant message identical, every other message in the same +role (a tool result answering the same call) and differing only in volatile runs such as +an id, a hostname, a clock time, or a duration. A request no path answers goes to the +live model, and so does everything after it, since its prompt carries the live reply.""" + +import difflib +import json +import re from collections import Counter from dataclasses import dataclass, field from typing import Literal @@ -21,38 +26,76 @@ ReplayMatch = Literal["exact", "structural"] -@dataclass +@dataclass(eq=False) class _Recording: trace: Trace children: dict[int | None, list[int]] = field(default_factory=dict) hashes: list[str] = field(default_factory=list) used: Counter[int] = field(default_factory=Counter) + adopted: bool = False + _masked: dict[int, str] = field(default_factory=dict) + _toolsets: dict[int, str] = field(default_factory=dict) def __post_init__(self) -> None: for node_id, node in enumerate(self.trace.nodes): self.children.setdefault(node.parent, []).append(node_id) self.hashes.append(graph.message_hash(node.message)) + def toolset(self, root: int) -> str: + if root not in self._toolsets: + self._toolsets[root] = _toolset(self.trace.nodes[root].tools) + return self._toolsets[root] + + def masked(self, node: int) -> str: + if node not in self._masked: + self._masked[node] = _volatile_masked(_text(self.trace.nodes[node].message)) + return self._masked[node] + + def next_reply(self, parent: int) -> int | None: + """The parent's first recorded reply not yet replayed.""" + replies = [ + n for n in self.children.get(parent, []) if self.trace.nodes[n].sampled + ] + used = self.used[parent] + return replies[used] if used < len(replies) else None + class Replay: def __init__(self, traces: list[Trace]) -> None: - self._unclaimed = list(traces) - self._claimed: dict[str, _Recording] = {} - - def claim(self, agent: str, task_key: str) -> str | None: - """The recorded trace id a newly minted trace of `agent` on `task_key` continues.""" - for trace in self._unclaimed: - if trace.agent.name == agent and trace.task.key == task_key: - self._unclaimed.remove(trace) - self._claimed[trace.id] = _Recording(trace) - return trace.id - return None + self._recordings = [_Recording(trace) for trace in traces] + self._bound: dict[str, _Recording] = {} + + def claim(self, agent: str, task_key: str, trace_id: str) -> str: + """Bind a newly minted trace of `agent` on `task_key` to a free recording and + return the id it runs under: the recording's own id the first time the recording + is claimed, else (a retry after a failed attempt) the trace's own.""" + bound = {id(recording) for recording in self._bound.values()} + free = [ + recording + for recording in self._recordings + if id(recording) not in bound + and recording.trace.agent.name == agent + and recording.trace.task.key == task_key + ] + if not free: + return trace_id + recording = min(free, key=lambda recording: recording.adopted) + recording.used.clear() + if not recording.adopted: + recording.adopted = True + trace_id = recording.trace.id + self._bound[trace_id] = recording + return trace_id + + def release(self, trace_id: str) -> None: + """Free a failed trace's recording for the attempt that retries it.""" + self._bound.pop(trace_id, None) def take( self, trace_id: str, prompt: list[Message], tools: list[Tool] | None ) -> tuple[Response, ReplayMatch] | None: """The recorded reply to `prompt` and how the prompt matched, or None to sample live.""" - recording = self._claimed.get(trace_id) + recording = self._bound.get(trace_id) if not prompt or recording is None: return None source = recording.trace @@ -64,12 +107,9 @@ def take( match = "structural" else: return None - replies = [ - n for n in recording.children.get(parent, []) if source.nodes[n].sampled - ] - if recording.used[parent] >= len(replies): + node = recording.next_reply(parent) + if node is None: return None - node = replies[recording.used[parent]] recording.used[parent] += 1 call = next((c for c in source.calls if c.node == node), None) response = Response( @@ -86,35 +126,53 @@ def take( def _structural_match( recording: _Recording, prompt: list[Message], tools: list[Tool] | None ) -> int | None: + """The recorded node the prompt ends at, matched by structure. Every branch that fits + the prompt so far stays a candidate, so siblings that agree on a long prefix resolve + at the message that tells them apart. Among the full matches that still have a reply, + the one with the most exact messages, then the most similar text, wins.""" source = recording.trace - parent: int | None = None + toolset = _toolset(tools) + frontier: dict[int | None, tuple[int, float]] = {None: (0, 0.0)} for message in prompt: - options = recording.children.get(parent, []) - if parent is None: - options = [n for n in options if source.nodes[n].tools == list(tools or [])] key = graph.message_hash(message) - exact = [n for n in options if recording.hashes[n] == key] - if exact: - parent = exact[0] - continue - if isinstance(message, AssistantMessage): + exact: dict[int | None, tuple[int, float]] = {} + alike: list[tuple[int, tuple[int, float]]] = [] + for parent, score in frontier.items(): + for n in recording.children.get(parent, []): + if parent is None and recording.toolset(n) != toolset: + continue + if recording.hashes[n] == key: + exact[n] = (score[0] + 1, score[1]) + elif _same_structure(source.nodes[n].message, message): + alike.append((n, score)) + frontier = exact + if alike: + text = _text(message) + masked = _volatile_masked(text) + alike = [(n, score) for n, score in alike if recording.masked(n) == masked] + for n, (matched, similarity) in alike: + if len(alike) > 1: + recorded = _text(source.nodes[n].message) + similarity += difflib.SequenceMatcher(None, recorded, text).ratio() + frontier[n] = (matched, similarity) + if not frontier: return None - alike = [ - n - for n in options - if source.nodes[n].message.role == message.role - and ( - not isinstance(message, ToolMessage) - or source.nodes[n].message.tool_call_id == message.tool_call_id - ) - ] - if not alike: - return None - text = _text(message) - parent = max( - alike, key=lambda n: _overlap(_text(source.nodes[n].message), text) - ) - return parent + answered = [ + n for n in frontier if n is not None and recording.next_reply(n) is not None + ] + return max(answered, key=lambda n: frontier[n]) if answered else None + + +def _same_structure(recorded: Message, message: Message) -> bool: + """Whether a recorded message may stand for `message` when only volatile text differs: + never an assistant message (the model's own words must match exactly), else the same + role, and for a tool result the same call.""" + if isinstance(message, AssistantMessage) or recorded.role != message.role: + return False + return not isinstance(message, ToolMessage) or ( + isinstance(recorded, ToolMessage) + and recorded.tool_call_id == message.tool_call_id + ) def _text(message: Message) -> str: @@ -123,13 +181,24 @@ def _text(message: Message) -> str: return "".join(getattr(part, "text", "") for part in message.content or []) -def _overlap(a: str, b: str) -> int: - """Shared leading plus trailing characters: how alike two variants of one message are.""" - head = next( - (i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b)) - ) - tail = next( - (i for i, (x, y) in enumerate(zip(reversed(a), reversed(b))) if x != y), - min(len(a), len(b)), +_VOLATILE = re.compile( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}" # uuids + r"|\b(?=[0-9a-fA-F]*[a-fA-F])(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{8,}\b" # hashes, hostnames + r"|\b\d+(?:\.\d+)?\s?(?:ns|us|µs|ms|s|secs?|seconds?)\b" # durations + r"|\b\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?\b" # clock times + r"|\b\d{10,}\b" # epoch timestamps +) +"""Runs that differ between two runs of one rollout without changing what it observed.""" + + +def _toolset(tools: list[Tool] | None) -> str: + """The tools a prompt declares, in any order, with volatile runs masked.""" + declared = sorted( + json.dumps(tool.model_dump(mode="json", exclude_none=True), sort_keys=True) + for tool in tools or [] ) - return head + tail + return _volatile_masked("\n".join(declared)) + + +def _volatile_masked(text: str) -> str: + return _VOLATILE.sub("#", text) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index ae5138fa53..16dadf2f07 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -97,10 +97,8 @@ def __init__( # this trace can be reproduced with. agent=AgentInfo(config=agent_config, name=agent_name), ) - if ctx.replay is not None and ( - claimed := ctx.replay.claim(agent_name, task.key) - ): - self.trace.id = claimed + if ctx.replay is not None: + self.trace.id = ctx.replay.claim(agent_name, task.key, self.trace.id) if on_trace is not None: on_trace(self.trace) interceptors = [ @@ -539,6 +537,8 @@ async def close(self) -> Trace: await self._stack.aclose() trace.is_completed = True trace.ok = not self._failed + if not trace.ok and self.ctx.replay is not None: + self.ctx.replay.release(trace.id) now = time.time() for span in ( trace.timing.boot, From 31db3956e9fdb0e10394d47fcb9a4aa041756697 Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Fri, 2 Oct 2026 23:35:26 +0000 Subject: [PATCH 3/8] Find each replayed reply by the prompt's last assistant message; add verbatim replay Each request now looks up the recorded reply that followed its last assistant message, then (resume) compares the request with that one recorded prompt: identical, or differing only in volatile runs, with images compared exactly. This replaces the tree search and matches the same replies. Verbatim replay (Replay(..., verbatim=True)) skips the comparison, never samples live, stops the rollout with replay_ended when the recording runs out, and claims a recording by agent alone, so a recorded seat can drive a seat of another task. Hex ids delimited by -, _ or / are masked as volatile. --- verifiers/v1/interception/server.py | 15 +- verifiers/v1/replay.py | 236 ++++++++++++++-------------- verifiers/v1/trace.py | 5 +- 3 files changed, 135 insertions(+), 121 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 94e043f92f..190539d828 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -95,7 +95,7 @@ def _replay_reply( session: RolloutSession, dialect: Dialect, request: Request -) -> tuple[Response, Literal["exact", "structural"]] | None: +) -> tuple[Response, Literal["exact", "structural", "verbatim"]] | None: """The recorded reply that answers this request, rendered for the harness; None to sample live. Replay only saves work, so a failure in it falls back to the model.""" if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig): @@ -551,7 +551,7 @@ def record_call( error: BaseException | None = None, policy_paths: list[str] | None = None, acp: ACPInfo | None = None, - replayed: Literal["exact", "structural"] | None = None, + replayed: Literal["exact", "structural", "verbatim"] | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): the model + effective settings that went upstream, timing, and — when the call @@ -803,6 +803,17 @@ async def sample() -> web.Response: error: Exception | None = None started = time.time() replayed = _replay_reply(session, dialect, model_request) + if ( + replayed is None + and session.ctx.replay is not None + and session.ctx.replay.drives(session.trace.id) + ): + # A verbatim seat never samples live: its recording ended here. + turn.commit_prompt() + session.trace.stop("replay_ended") + return web.json_response( + dialect.error_body("rollout stopped: replay_ended"), status=400 + ) try: try: # What actually goes upstream: the native body with the rollout's model + diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py index 9f3f9fd705..92bfd4b35d 100644 --- a/verifiers/v1/replay.py +++ b/verifiers/v1/replay.py @@ -1,21 +1,24 @@ -"""Resume interrupted rollouts by replaying their recorded model replies. +"""Replay recorded model replies: resume an interrupted rollout, or drive a seat verbatim. -A rollout whose context carries a `Replay` claims a free recorded trace of its agent on -its task and continues under that trace's id. A failed attempt frees its recording, so -the attempt that retries it replays the recording again, under its own id. Each model -request is matched to a path of the recording and answered with the recorded reply; the -harness still runs every tool call, so its runtime and program state are rebuilt. +A rollout whose context carries a `Replay` claims a free recorded trace of its agent and +continues under that trace's id; a failed attempt frees its recording, so the attempt that +retries it replays the recording again, under its own id. Each model request is answered +with the recorded reply that followed the prompt's last assistant message (or opened a +branch, when the prompt has none). The harness still runs every tool call, so its runtime +and program state are rebuilt. -A prompt matches its exact recorded path, or else (`structural`) a path with the same -tools and structure: every assistant message identical, every other message in the same -role (a tool result answering the same call) and differing only in volatile runs such as -an id, a hostname, a clock time, or a duration. A request no path answers goes to the -live model, and so does everything after it, since its prompt carries the live reply.""" +Resume (the default) replays a reply only when the prompt is its recorded prompt: every +message identical (`exact`), or every assistant message identical and every other message +in the same role, answering the same call, and differing only in volatile runs such as an +id, a hostname, a clock time, or a duration (`structural`). Any other request samples live, +and so does everything after it, since its prompt carries the live reply. + +Verbatim replays the recording whatever the tools returned or other agents said, never +samples live, and stops the rollout (`replay_ended`) once the recording has no reply left. +It claims a recording by agent alone, so a recorded seat can drive a seat of another task.""" -import difflib import json import re -from collections import Counter from dataclasses import dataclass, field from typing import Literal @@ -23,45 +26,45 @@ from verifiers.v1.trace import Trace from verifiers.v1.types import AssistantMessage, Message, Response, Tool, ToolMessage -ReplayMatch = Literal["exact", "structural"] +ReplayMatch = Literal["exact", "structural", "verbatim"] @dataclass(eq=False) class _Recording: trace: Trace - children: dict[int | None, list[int]] = field(default_factory=dict) - hashes: list[str] = field(default_factory=list) - used: Counter[int] = field(default_factory=Counter) adopted: bool = False - _masked: dict[int, str] = field(default_factory=dict) - _toolsets: dict[int, str] = field(default_factory=dict) + used: set[int] = field(default_factory=set) + hashes: list[str] = field(default_factory=list) + replies: dict[str | None, list[int]] = field(default_factory=dict) + """Recorded replies by the hash of the assistant message their prompt ends after.""" def __post_init__(self) -> None: - for node_id, node in enumerate(self.trace.nodes): - self.children.setdefault(node.parent, []).append(node_id) - self.hashes.append(graph.message_hash(node.message)) - - def toolset(self, root: int) -> str: - if root not in self._toolsets: - self._toolsets[root] = _toolset(self.trace.nodes[root].tools) - return self._toolsets[root] - - def masked(self, node: int) -> str: - if node not in self._masked: - self._masked[node] = _volatile_masked(_text(self.trace.nodes[node].message)) - return self._masked[node] - - def next_reply(self, parent: int) -> int | None: - """The parent's first recorded reply not yet replayed.""" - replies = [ - n for n in self.children.get(parent, []) if self.trace.nodes[n].sampled - ] - used = self.used[parent] - return replies[used] if used < len(replies) else None + nodes = self.trace.nodes + self.hashes = [graph.message_hash(node.message) for node in nodes] + for reply, node in enumerate(nodes): + if not node.sampled: + continue + last = node.parent + while last is not None and not isinstance( + nodes[last].message, AssistantMessage + ): + last = nodes[last].parent + key = None if last is None else self.hashes[last] + self.replies.setdefault(key, []).append(reply) + + def prompt(self, reply: int) -> list[int]: + """The recorded prompt of `reply`, root first.""" + nodes: list[int] = [] + current = self.trace.nodes[reply].parent + while current is not None: + nodes.append(current) + current = self.trace.nodes[current].parent + return nodes[::-1] class Replay: - def __init__(self, traces: list[Trace]) -> None: + def __init__(self, traces: list[Trace], *, verbatim: bool = False) -> None: + self.verbatim = verbatim self._recordings = [_Recording(trace) for trace in traces] self._bound: dict[str, _Recording] = {} @@ -75,7 +78,7 @@ def claim(self, agent: str, task_key: str, trace_id: str) -> str: for recording in self._recordings if id(recording) not in bound and recording.trace.agent.name == agent - and recording.trace.task.key == task_key + and (self.verbatim or recording.trace.task.key == task_key) ] if not free: return trace_id @@ -91,88 +94,79 @@ def release(self, trace_id: str) -> None: """Free a failed trace's recording for the attempt that retries it.""" self._bound.pop(trace_id, None) + def drives(self, trace_id: str) -> bool: + """Whether this trace follows its recording verbatim, never sampling live.""" + return self.verbatim and trace_id in self._bound + def take( self, trace_id: str, prompt: list[Message], tools: list[Tool] | None ) -> tuple[Response, ReplayMatch] | None: - """The recorded reply to `prompt` and how the prompt matched, or None to sample live.""" + """The recorded reply to `prompt` and how the prompt matched, or None.""" recording = self._bound.get(trace_id) if not prompt or recording is None: return None - source = recording.trace - turn = graph.prepare_turn(source, prompt, tools) - match: ReplayMatch = "exact" - if len(turn.prefix_node_ids) == len(prompt): - parent = turn.prefix_node_ids[-1] - elif (parent := _structural_match(recording, prompt, tools)) is not None: - match = "structural" + last = next( + (m for m in reversed(prompt) if isinstance(m, AssistantMessage)), None + ) + key = None if last is None else graph.message_hash(last) + candidates = [ + reply + for reply in recording.replies.get(key, []) + if reply not in recording.used + ] + matches = [ + (match, reply) + for reply in candidates + if (match := _compare(recording, reply, prompt, tools)) is not None + ] + if matches: + match, reply = min(matches, key=lambda pair: pair[0] != "exact") + elif self.verbatim and candidates: + match, reply = "verbatim", candidates[0] else: return None - node = recording.next_reply(parent) - if node is None: - return None - recording.used[parent] += 1 - call = next((c for c in source.calls if c.node == node), None) + recording.used.add(reply) + source = recording.trace + call = next((c for c in source.calls if c.node == reply), None) response = Response( - id=f"replay-{node}", + id=f"replay-{reply}", created=0, model=call.model if call and call.model else "", - message=source.nodes[node].message.model_copy(deep=True), + message=source.nodes[reply].message.model_copy(deep=True), finish_reason=call.finish_reason if call else None, usage=call.usage if call else None, ) - return response, match - - -def _structural_match( - recording: _Recording, prompt: list[Message], tools: list[Tool] | None -) -> int | None: - """The recorded node the prompt ends at, matched by structure. Every branch that fits - the prompt so far stays a candidate, so siblings that agree on a long prefix resolve - at the message that tells them apart. Among the full matches that still have a reply, - the one with the most exact messages, then the most similar text, wins.""" - source = recording.trace - toolset = _toolset(tools) - frontier: dict[int | None, tuple[int, float]] = {None: (0, 0.0)} - for message in prompt: - key = graph.message_hash(message) - exact: dict[int | None, tuple[int, float]] = {} - alike: list[tuple[int, tuple[int, float]]] = [] - for parent, score in frontier.items(): - for n in recording.children.get(parent, []): - if parent is None and recording.toolset(n) != toolset: - continue - if recording.hashes[n] == key: - exact[n] = (score[0] + 1, score[1]) - elif _same_structure(source.nodes[n].message, message): - alike.append((n, score)) - frontier = exact - if alike: - text = _text(message) - masked = _volatile_masked(text) - alike = [(n, score) for n, score in alike if recording.masked(n) == masked] - for n, (matched, similarity) in alike: - if len(alike) > 1: - recorded = _text(source.nodes[n].message) - similarity += difflib.SequenceMatcher(None, recorded, text).ratio() - frontier[n] = (matched, similarity) - if not frontier: + return response, "verbatim" if self.verbatim else match + + +def _compare( + recording: _Recording, reply: int, prompt: list[Message], tools: list[Tool] | None +) -> Literal["exact", "structural"] | None: + """How `prompt` matches the recorded prompt of `reply`, or None when it does not.""" + nodes = recording.prompt(reply) + if len(nodes) != len(prompt): + return None + recorded_tools = recording.trace.nodes[nodes[0]].tools + exact = recorded_tools == list(tools or []) + if not exact and _toolset(recorded_tools) != _toolset(tools): + return None + for node, message in zip(nodes, prompt): + if recording.hashes[node] == graph.message_hash(message): + continue + exact = False + recorded = recording.trace.nodes[node].message + if ( + isinstance(message, AssistantMessage) + or recorded.role != message.role + or ( + isinstance(message, ToolMessage) + and getattr(recorded, "tool_call_id", None) != message.tool_call_id + ) + or _media(recorded) != _media(message) + or _volatile_masked(_text(recorded)) != _volatile_masked(_text(message)) + ): return None - answered = [ - n for n in frontier if n is not None and recording.next_reply(n) is not None - ] - return max(answered, key=lambda n: frontier[n]) if answered else None - - -def _same_structure(recorded: Message, message: Message) -> bool: - """Whether a recorded message may stand for `message` when only volatile text differs: - never an assistant message (the model's own words must match exactly), else the same - role, and for a tool result the same call.""" - if isinstance(message, AssistantMessage) or recorded.role != message.role: - return False - return not isinstance(message, ToolMessage) or ( - isinstance(recorded, ToolMessage) - and recorded.tool_call_id == message.tool_call_id - ) + return "exact" if exact else "structural" def _text(message: Message) -> str: @@ -181,14 +175,11 @@ def _text(message: Message) -> str: return "".join(getattr(part, "text", "") for part in message.content or []) -_VOLATILE = re.compile( - r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}" # uuids - r"|\b(?=[0-9a-fA-F]*[a-fA-F])(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{8,}\b" # hashes, hostnames - r"|\b\d+(?:\.\d+)?\s?(?:ns|us|µs|ms|s|secs?|seconds?)\b" # durations - r"|\b\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?\b" # clock times - r"|\b\d{10,}\b" # epoch timestamps -) -"""Runs that differ between two runs of one rollout without changing what it observed.""" +def _media(message: Message) -> list[str]: + """A message's non-text parts (images), which must match exactly.""" + if isinstance(message.content, str): + return [] + return [part.model_dump_json() for part in message.content if part.type != "text"] def _toolset(tools: list[Tool] | None) -> str: @@ -200,5 +191,16 @@ def _toolset(tools: list[Tool] | None) -> str: return _volatile_masked("\n".join(declared)) +_VOLATILE = re.compile( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}" # uuids + r"|\b(?=[0-9a-fA-F]*[a-fA-F])(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{8,}\b" # hashes, hostnames + r"|(?<=[-_/])[0-9a-fA-F]{8,}\b" # ids in names and paths + r"|\b\d+(?:\.\d+)?\s?(?:ns|us|µs|ms|s|secs?|seconds?)\b" # durations + r"|\b\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?\b" # clock times + r"|\b\d{10,}\b" # epoch timestamps +) +"""Runs that differ between two runs of one rollout without changing what it observed.""" + + def _volatile_masked(text: str) -> str: return _VOLATILE.sub("#", text) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index a8ffe75b67..b0706dad44 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -173,9 +173,10 @@ class ModelCall(BaseModel): """Policy mediation applied to the request before this call.""" acp: ACPInfo | None = None """Metadata advertised by the ACP harness for this model request.""" - replayed: Literal["exact", "structural"] | None = None + replayed: Literal["exact", "structural", "verbatim"] | None = None """Answered from a recorded trace (`ModelContext.replay`) instead of the provider, and - whether the prompt matched its recorded path exactly or by structure.""" + how: the prompt matched its recorded prompt exactly or by structure, or the seat + follows its recording verbatim.""" def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall, int]]: From 4d95d26ebe9495081f27b7481952148db35ee192 Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Sat, 3 Oct 2026 00:22:05 +0000 Subject: [PATCH 4/8] Replay only exact prompts on resume; name the modes resume and playback Resume replays a recorded reply only while the prompt is exactly its recorded prompt and samples live from the first prompt that differs. Playback (formerly verbatim) replays every reply whatever the prompt and tags a reply whose prompt differs as forced. The volatile-text masking and similarity matching are gone. --- verifiers/v1/interception/server.py | 8 +- verifiers/v1/replay.py | 151 ++++++++-------------------- verifiers/v1/trace.py | 8 +- 3 files changed, 50 insertions(+), 117 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 190539d828..84b21c3646 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -95,7 +95,7 @@ def _replay_reply( session: RolloutSession, dialect: Dialect, request: Request -) -> tuple[Response, Literal["exact", "structural", "verbatim"]] | None: +) -> tuple[Response, Literal["exact", "forced"]] | None: """The recorded reply that answers this request, rendered for the harness; None to sample live. Replay only saves work, so a failure in it falls back to the model.""" if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig): @@ -551,7 +551,7 @@ def record_call( error: BaseException | None = None, policy_paths: list[str] | None = None, acp: ACPInfo | None = None, - replayed: Literal["exact", "structural", "verbatim"] | None = None, + replayed: Literal["exact", "forced"] | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): the model + effective settings that went upstream, timing, and — when the call @@ -806,9 +806,9 @@ async def sample() -> web.Response: if ( replayed is None and session.ctx.replay is not None - and session.ctx.replay.drives(session.trace.id) + and session.ctx.replay.plays_back(session.trace.id) ): - # A verbatim seat never samples live: its recording ended here. + # A playback never samples live: its recording ended here. turn.commit_prompt() session.trace.stop("replay_ended") return web.json_response( diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py index 92bfd4b35d..92d34cd725 100644 --- a/verifiers/v1/replay.py +++ b/verifiers/v1/replay.py @@ -1,32 +1,30 @@ -"""Replay recorded model replies: resume an interrupted rollout, or drive a seat verbatim. - -A rollout whose context carries a `Replay` claims a free recorded trace of its agent and -continues under that trace's id; a failed attempt frees its recording, so the attempt that -retries it replays the recording again, under its own id. Each model request is answered -with the recorded reply that followed the prompt's last assistant message (or opened a -branch, when the prompt has none). The harness still runs every tool call, so its runtime -and program state are rebuilt. - -Resume (the default) replays a reply only when the prompt is its recorded prompt: every -message identical (`exact`), or every assistant message identical and every other message -in the same role, answering the same call, and differing only in volatile runs such as an -id, a hostname, a clock time, or a duration (`structural`). Any other request samples live, -and so does everything after it, since its prompt carries the live reply. - -Verbatim replays the recording whatever the tools returned or other agents said, never -samples live, and stops the rollout (`replay_ended`) once the recording has no reply left. -It claims a recording by agent alone, so a recorded seat can drive a seat of another task.""" - -import json -import re +"""Replay recorded model replies, in one of two modes. + +`resume` (the default) continues an interrupted rollout: it replays recorded replies while +each prompt is exactly its recorded prompt (`exact`), then samples live from the first +prompt that differs. + +`playback` plays a recording back, on any task: it replays every recorded reply whatever the +tools returned or other agents said, never samples live, and stops the rollout +(`replay_ended`) when the recording runs out. A reply played onto a prompt that differs from +its recorded prompt is `forced`: the recorded model never saw that prompt. + +A rollout whose context carries a `Replay` claims a free recording of its agent (on its task, +in resume) and runs under the recording's trace id. A failed attempt frees its recording, so +the attempt that retries it replays the recording again, under its own id. Each model +request is answered with the recorded reply that followed the prompt's last assistant +message, or that opened a branch when the prompt has none. The harness still runs every tool +call, so its runtime and program state are rebuilt.""" + from dataclasses import dataclass, field from typing import Literal from verifiers.v1 import graph from verifiers.v1.trace import Trace -from verifiers.v1.types import AssistantMessage, Message, Response, Tool, ToolMessage +from verifiers.v1.types import AssistantMessage, Message, Response, Tool -ReplayMatch = Literal["exact", "structural", "verbatim"] +ReplayMode = Literal["resume", "playback"] +ReplayMatch = Literal["exact", "forced"] @dataclass(eq=False) @@ -52,19 +50,22 @@ def __post_init__(self) -> None: key = None if last is None else self.hashes[last] self.replies.setdefault(key, []).append(reply) - def prompt(self, reply: int) -> list[int]: - """The recorded prompt of `reply`, root first.""" + def is_prompt(self, reply: int, hashes: list[str], tools: list[Tool]) -> bool: + """Whether a prompt with these message hashes and tools is `reply`'s recorded prompt.""" nodes: list[int] = [] current = self.trace.nodes[reply].parent while current is not None: nodes.append(current) current = self.trace.nodes[current].parent - return nodes[::-1] + nodes.reverse() + return [self.hashes[n] for n in nodes] == hashes and self.trace.nodes[ + nodes[0] + ].tools == tools class Replay: - def __init__(self, traces: list[Trace], *, verbatim: bool = False) -> None: - self.verbatim = verbatim + def __init__(self, traces: list[Trace], *, mode: ReplayMode = "resume") -> None: + self.mode = mode self._recordings = [_Recording(trace) for trace in traces] self._bound: dict[str, _Recording] = {} @@ -78,7 +79,7 @@ def claim(self, agent: str, task_key: str, trace_id: str) -> str: for recording in self._recordings if id(recording) not in bound and recording.trace.agent.name == agent - and (self.verbatim or recording.trace.task.key == task_key) + and (self.mode == "playback" or recording.trace.task.key == task_key) ] if not free: return trace_id @@ -94,14 +95,14 @@ def release(self, trace_id: str) -> None: """Free a failed trace's recording for the attempt that retries it.""" self._bound.pop(trace_id, None) - def drives(self, trace_id: str) -> bool: - """Whether this trace follows its recording verbatim, never sampling live.""" - return self.verbatim and trace_id in self._bound + def plays_back(self, trace_id: str) -> bool: + """Whether this trace plays its recording back, never sampling live.""" + return self.mode == "playback" and trace_id in self._bound def take( self, trace_id: str, prompt: list[Message], tools: list[Tool] | None ) -> tuple[Response, ReplayMatch] | None: - """The recorded reply to `prompt` and how the prompt matched, or None.""" + """The recorded reply for `prompt` and how it was matched, or None to sample live.""" recording = self._bound.get(trace_id) if not prompt or recording is None: return None @@ -114,15 +115,14 @@ def take( for reply in recording.replies.get(key, []) if reply not in recording.used ] - matches = [ - (match, reply) - for reply in candidates - if (match := _compare(recording, reply, prompt, tools)) is not None + hashes = [graph.message_hash(message) for message in prompt] + exact = [ + r for r in candidates if recording.is_prompt(r, hashes, list(tools or [])) ] - if matches: - match, reply = min(matches, key=lambda pair: pair[0] != "exact") - elif self.verbatim and candidates: - match, reply = "verbatim", candidates[0] + if exact: + reply, match = exact[0], "exact" + elif self.mode == "playback" and candidates: + reply, match = candidates[0], "forced" else: return None recording.used.add(reply) @@ -136,71 +136,4 @@ def take( finish_reason=call.finish_reason if call else None, usage=call.usage if call else None, ) - return response, "verbatim" if self.verbatim else match - - -def _compare( - recording: _Recording, reply: int, prompt: list[Message], tools: list[Tool] | None -) -> Literal["exact", "structural"] | None: - """How `prompt` matches the recorded prompt of `reply`, or None when it does not.""" - nodes = recording.prompt(reply) - if len(nodes) != len(prompt): - return None - recorded_tools = recording.trace.nodes[nodes[0]].tools - exact = recorded_tools == list(tools or []) - if not exact and _toolset(recorded_tools) != _toolset(tools): - return None - for node, message in zip(nodes, prompt): - if recording.hashes[node] == graph.message_hash(message): - continue - exact = False - recorded = recording.trace.nodes[node].message - if ( - isinstance(message, AssistantMessage) - or recorded.role != message.role - or ( - isinstance(message, ToolMessage) - and getattr(recorded, "tool_call_id", None) != message.tool_call_id - ) - or _media(recorded) != _media(message) - or _volatile_masked(_text(recorded)) != _volatile_masked(_text(message)) - ): - return None - return "exact" if exact else "structural" - - -def _text(message: Message) -> str: - if isinstance(message.content, str): - return message.content - return "".join(getattr(part, "text", "") for part in message.content or []) - - -def _media(message: Message) -> list[str]: - """A message's non-text parts (images), which must match exactly.""" - if isinstance(message.content, str): - return [] - return [part.model_dump_json() for part in message.content if part.type != "text"] - - -def _toolset(tools: list[Tool] | None) -> str: - """The tools a prompt declares, in any order, with volatile runs masked.""" - declared = sorted( - json.dumps(tool.model_dump(mode="json", exclude_none=True), sort_keys=True) - for tool in tools or [] - ) - return _volatile_masked("\n".join(declared)) - - -_VOLATILE = re.compile( - r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}" # uuids - r"|\b(?=[0-9a-fA-F]*[a-fA-F])(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{8,}\b" # hashes, hostnames - r"|(?<=[-_/])[0-9a-fA-F]{8,}\b" # ids in names and paths - r"|\b\d+(?:\.\d+)?\s?(?:ns|us|µs|ms|s|secs?|seconds?)\b" # durations - r"|\b\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?\b" # clock times - r"|\b\d{10,}\b" # epoch timestamps -) -"""Runs that differ between two runs of one rollout without changing what it observed.""" - - -def _volatile_masked(text: str) -> str: - return _VOLATILE.sub("#", text) + return response, match diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 00c53854eb..e84e1184e8 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -173,10 +173,10 @@ class ModelCall(BaseModel): """Policy mediation applied to the request before this call.""" acp: ACPInfo | None = None """Metadata advertised by the ACP harness for this model request.""" - replayed: Literal["exact", "structural", "verbatim"] | None = None - """Answered from a recorded trace (`ModelContext.replay`) instead of the provider, and - how: the prompt matched its recorded prompt exactly or by structure, or the seat - follows its recording verbatim.""" + replayed: Literal["exact", "forced"] | None = None + """Answered from a recorded trace (`ModelContext.replay`) instead of the provider: the + prompt was exactly its recorded prompt, or a playback forced the reply onto a prompt that + differs from it.""" def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall, int]]: From 81182a7eecfe9a942bc947c069684cdd0ec18a9f Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Sat, 3 Oct 2026 17:04:44 +0000 Subject: [PATCH 5/8] Reject replay playback on the train client --- verifiers/v1/interception/server.py | 4 ++-- verifiers/v1/rollout.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 84b21c3646..7ea4d6a7c1 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -96,8 +96,8 @@ def _replay_reply( session: RolloutSession, dialect: Dialect, request: Request ) -> tuple[Response, Literal["exact", "forced"]] | None: - """The recorded reply that answers this request, rendered for the harness; None to - sample live. Replay only saves work, so a failure in it falls back to the model.""" + """The recorded reply that answers this request, rendered for the harness; None when + there is none, or when the lookup fails (logged).""" if session.ctx.replay is None or isinstance(session.ctx.client, TrainClientConfig): return None try: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 57fed12798..7be1bce4f4 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -10,6 +10,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import AgentConfig +from verifiers.v1.configs.client import TrainClientConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import ( HarnessError, @@ -132,6 +133,10 @@ def __init__( agent=AgentInfo(config=agent_config, name=agent_name), ) if ctx.replay is not None: + if ctx.replay.mode == "playback" and isinstance( + ctx.client, TrainClientConfig + ): + raise ValueError("replay playback cannot run on the train client") self.trace.id = ctx.replay.claim(agent_name, task.key, self.trace.id) if on_trace is not None: on_trace(self.trace) From 8214a89dbf607b8d36ea95d8e9417b95ca8dc613 Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Sat, 3 Oct 2026 17:06:36 +0000 Subject: [PATCH 6/8] Drop the playback train-client check --- verifiers/v1/rollout.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 7be1bce4f4..57fed12798 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -10,7 +10,6 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import AgentConfig -from verifiers.v1.configs.client import TrainClientConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import ( HarnessError, @@ -133,10 +132,6 @@ def __init__( agent=AgentInfo(config=agent_config, name=agent_name), ) if ctx.replay is not None: - if ctx.replay.mode == "playback" and isinstance( - ctx.client, TrainClientConfig - ): - raise ValueError("replay playback cannot run on the train client") self.trace.id = ctx.replay.claim(agent_name, task.key, self.trace.id) if on_trace is not None: on_trace(self.trace) From e61bfc7129227ca9f904107952050f7643f42cf7 Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Sat, 3 Oct 2026 18:37:36 +0000 Subject: [PATCH 7/8] Document trace replay --- docs/v1/evaluation.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/v1/evaluation.md b/docs/v1/evaluation.md index f1155759aa..a94141c9a3 100644 --- a/docs/v1/evaluation.md +++ b/docs/v1/evaluation.md @@ -53,7 +53,21 @@ The output from evaluations are written into `outputs/----/ ## Resuming evaluations -`--resume ` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. +`--resume ` re-runs only the rollouts a previous run left missing or errored, appending to that run's own `traces.jsonl`. It reloads the run's saved `config.toml` verbatim, so it takes no other arguments. Good rollouts are kept, while errored ones are dropped and redone. Rollouts the interruption cut off run again from the start; prime-rl's `uv run eval --resume` replays them from their live traces instead. + +## Replaying traces + +`vf.Replay` answers model calls with the replies in recorded traces. The harness runs every replayed tool call again, so it rebuilds the recorded rollout's files, processes, and tool state. It has two modes: + +- `resume` (default): replay while each prompt equals its recorded prompt, then sample live. This continues an interrupted rollout. +- `playback`: replay every recorded reply whatever the prompt, never sample live, and stop with `replay_ended` when the recording runs out. This reproduces a trace's deliverables, or drives one seat of a multi-agent env. + +```python +async with vf.Agent(config, replay=vf.Replay(traces)) as agent: + trace = await agent.run(task) +``` + +An env plays back one seat from `setup()`: `agents.solver.ctx = dataclasses.replace(agents.solver.ctx, replay=vf.Replay(traces, mode="playback"))`. `ModelCall.replayed` tags each replayed call: `exact` when its prompt equals the recorded prompt, `forced` when a playback serves it to a prompt that differs. Tool calls run again, side effects included. Replay is for evaluation only: the train client needs token ids. ## Disabling tools From 22cb2bebce73c60f2871f4deca2b16f49978573b Mon Sep 17 00:00:00 2001 From: Parker Pettit Date: Sat, 3 Oct 2026 20:30:44 +0000 Subject: [PATCH 8/8] Skip recordings that do not load A live trace that lost a delta could raise while its replay was built, failing the resumed rollout on every later resume instead of running it live. --- verifiers/v1/replay.py | 14 +++++++++++++- verifiers/v1/serve/server.py | 9 ++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/replay.py b/verifiers/v1/replay.py index 92d34cd725..849670bb9e 100644 --- a/verifiers/v1/replay.py +++ b/verifiers/v1/replay.py @@ -16,6 +16,7 @@ message, or that opened a branch when the prompt has none. The harness still runs every tool call, so its runtime and program state are rebuilt.""" +import logging from dataclasses import dataclass, field from typing import Literal @@ -23,6 +24,8 @@ from verifiers.v1.trace import Trace from verifiers.v1.types import AssistantMessage, Message, Response, Tool +logger = logging.getLogger(__name__) + ReplayMode = Literal["resume", "playback"] ReplayMatch = Literal["exact", "forced"] @@ -66,7 +69,16 @@ def is_prompt(self, reply: int, hashes: list[str], tools: list[Tool]) -> bool: class Replay: def __init__(self, traces: list[Trace], *, mode: ReplayMode = "resume") -> None: self.mode = mode - self._recordings = [_Recording(trace) for trace in traces] + self._recordings: list[_Recording] = [] + for trace in traces: + try: + self._recordings.append(_Recording(trace)) + except Exception: + logger.warning( + "skipping a recording that does not load: id=%s", + trace.id, + exc_info=True, + ) self._bound: dict[str, _Recording] = {} def claim(self, agent: str, task_key: str, trace_id: str) -> str: diff --git a/verifiers/v1/serve/server.py b/verifiers/v1/serve/server.py index e89aa6783f..1bb65f525b 100644 --- a/verifiers/v1/serve/server.py +++ b/verifiers/v1/serve/server.py @@ -102,7 +102,14 @@ async def _run( ) -> RunResponse: ctx = self._context(req.client, req.model, req.sampling) if req.replay: - traces = [WireTrace.model_validate(trace) for trace in req.replay] + traces = [] + for trace in req.replay: + try: + traces.append(WireTrace.model_validate(trace)) + except ValueError: + logger.warning( + "skipping a recording that does not validate", exc_info=True + ) ctx = replace(ctx, replay=Replay(traces)) (slot,) = self.env.slots(self._build_task(req.task_data))