diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..9424ac67c2 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -7,9 +7,14 @@ import pytest import verifiers.v1 as vf +from verifiers.v1 import graph from verifiers.v1.agent import Interaction +from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute +from verifiers.v1.errors import HarnessError from verifiers.v1.graph import MessageNode +from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.rollout import Rollout, RolloutTimeouts +from verifiers.v1.session import RolloutSession from verifiers.v1.types import AssistantMessage @@ -33,6 +38,109 @@ def make_trace() -> vf.Trace: ) +async def test_each_sample_keeps_its_tool_result_across_assistant_replays(): + replacements = iter( + ["First result", "Second result", "Follow-up result", "Third result"] + ) + + def replace_result(request: vf.Request) -> vf.Request | None: + tool = request.messages[-1] + if isinstance(tool, vf.ToolMessage) and tool.content == "": + tool.content = next(replacements) + return request + return None + + trace = make_trace() + trace._replay_view = lambda message, dialect: ( + drop_blank_text_blocks(message) if dialect == "anthropic_messages" else message + ) + sampled = AssistantMessage( + content="Ready\n", + text_block_offsets=[5], + tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")], + ) + session = RolloutSession( + ModelRoute(ModelContext(model="test", client=EvalClientConfig())), + trace, + request_interceptors=[replace_result], + gates_tools=True, + ) + replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) + other = replay.model_copy(update={"content": "Another action"}) + prompt = [vf.UserMessage(content="go")] + + def commit(messages, assistant, tools=None): + turn = graph.prepare_turn( + trace, messages, tools=tools, dialect="anthropic_messages" + ) + return turn.commit( + vf.Response( + id="", created=0, model="test", message=assistant, finish_reason="stop" + ) + ) + + # The two samples reuse a call ID, but their replayed messages identify them. + catalogs = [[vf.Tool(name=name)] for name in ("filtered_first", "filtered_second")] + for assistant, replayed, expected, tools in [ + (sampled, replay, "First result", catalogs[0]), + (other, other, "Second result", catalogs[1]), + ]: + node = commit(prompt, assistant) + await session.gate_tool_calls(node, "anthropic_messages") + assert (await session.decide_tool("call"))["action"] == "deny" + request = vf.Request( + messages=[ + *prompt, + replayed, + vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), + ], + tools=tools, + ) + rewritten, _, _ = await session.rewrite_request( + request, dialect="anthropic_messages" + ) + assert rewritten.messages[-1].content == expected + if assistant is sampled: + first_request = request + first_path = graph.path_node_ids(trace, node) + commit(rewritten.messages, AssistantMessage(content="done")) + + # The next request can change its available tools independently of its history. + changed_catalog = first_request.model_copy(update={"tools": catalogs[1]}) + earlier, _, _ = await session.rewrite_request( + changed_catalog, dialect="anthropic_messages" + ) + assert earlier.messages[-1].content == "First result" + + # An unrelated branch adds a tool to the trace-wide catalog. The next gate + # still knows exactly which sampled ancestors issued its historical calls. + commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")]) + follow_up = AssistantMessage( + tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")] + ) + node = commit(rewritten.messages, follow_up) + await session.gate_tool_calls(node, "anthropic_messages") + verdict = await session.decide_tool("next") + assert verdict["message"]["content"] == "Follow-up result" + + # The third sample reuses the first sample's replay and call ID under a new catalog. + node = commit(prompt, sampled) + await session.gate_tool_calls(node, "anthropic_messages") + assert (await session.decide_tool("call"))["message"]["content"] == "Third result" + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + await session.rewrite_request(changed_catalog, dialect="anthropic_messages") + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + await session.rewrite_request(first_request, dialect="anthropic_messages") + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + session.pin_tool_results( + first_request.model_copy(update={"tools": None}), "anthropic_messages" + ) + restored = session.pin_tool_results( + changed_catalog, "anthropic_messages", first_path + ) + assert restored.messages[-1].content == "First result" + + def make_rollout(trace: vf.Trace, session, time_remaining=None): run = FailingSegmentRollout() run.trace = trace diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index e47fd68e0c..90c6c897d0 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -414,7 +414,9 @@ def _tools_hash(tools: list[Tool] | None) -> str: return hashlib.blake2b(data, digest_size=16).hexdigest() -def _message_key(trace: Trace, message: Message, dialect: DialectName | None) -> str: +def replay_message_hash( + trace: Trace, message: Message, dialect: DialectName | None +) -> str: """`message_hash` of the message as the trace's harness replays it over `dialect`, so a turn the harness re-sends with cosmetic rewrites still matches the node it came from.""" @@ -433,7 +435,7 @@ def _node_key( return ( parent, _tools_hash(tools) if parent is None else None, - _message_key(trace, message, dialect), + replay_message_hash(trace, message, dialect), ) @@ -472,12 +474,12 @@ def message_prefix_len( parents: list[int | None] = [None] matched = 0 for message in prompt: - key = _message_key(trace, message, dialect) + key = replay_message_hash(trace, message, dialect) parents = [ node_id for parent in parents for node_id in children.get(parent, []) - if _message_key(trace, trace.nodes[node_id].message, dialect) == key + if replay_message_hash(trace, trace.nodes[node_id].message, dialect) == key ] if not parents: break @@ -743,7 +745,7 @@ def prepare_turn( if len(children) == 1 and trace.nodes[children[0]].message == msg: existing = children[0] if existing is None: - message_key = _message_key(trace, msg, dialect) + message_key = replay_message_hash(trace, msg, dialect) existing = idx.get( (parent, tools_hash if parent is None else None, message_key) ) @@ -1140,12 +1142,17 @@ def leaves(trace: Trace) -> list[int]: return [i for i in range(len(trace.nodes)) if i not in has_child] -def path(trace: Trace, node: int) -> list[Message]: - """The conversation ending at `node`: its root-to-node messages.""" - messages: list[Message] = [] +def path_node_ids(trace: Trace, node: int) -> list[int]: + """The physical ancestry from the root through the given node.""" + nodes: list[int] = [] current: int | None = node while current is not None: - messages.append(trace.nodes[current].message) + nodes.append(current) current = trace.nodes[current].parent - messages.reverse() - return messages + nodes.reverse() + return nodes + + +def path(trace: Trace, node: int) -> list[Message]: + """The conversation ending at `node`: its root-to-node messages.""" + return [trace.nodes[i].message for i in path_node_ids(trace, node)] diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 95b831a1c2..9f44e9550e 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1013,7 +1013,7 @@ async def sample() -> web.Response: session.consume_prepared(turn.tail) session.trace.response_rewrites.extend(response_rewrites) if stopped is None: - stopped = await session.gate_tool_calls(node) + stopped = await session.gate_tool_calls(node, dialect.name) if stopped is not None: session.trace.stop(stopped) return _error_response( diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index b06d6f8a55..d25971f297 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -19,7 +19,7 @@ import inspect import json from collections import Counter -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from contextlib import AbstractAsyncContextManager from dataclasses import dataclass, field, replace from functools import cached_property @@ -54,6 +54,7 @@ logger = get_logger(__name__) MCPProxy = Callable[[str], AbstractAsyncContextManager[str]] +ToolResultKey = tuple[int | str, str] def hook_boundary(handler: Callable, *, allow_trace: bool) -> type: @@ -203,20 +204,22 @@ class RolloutSession: """Handler tasks currently serving this session. aiohttp does not cancel a handler when its client disconnects, so a request whose program died at teardown would keep driving the exchange (upstream call, simulator turn) — unregistering cancels these instead.""" - tool_decisions: dict[tuple[str, str], ToolMessage | None] = field( + tool_decisions: dict[ToolResultKey, ToolMessage | None] = field( default_factory=dict ) """The pre-execution verdict per proposed call occurrence: None to run it, else the result the policy put in its place — that call must not execute.""" - pinned_tool_results: dict[tuple[str, str], ToolMessage] = field( - default_factory=dict - ) + pinned_tool_results: dict[ToolResultKey, ToolMessage] = field(default_factory=dict) """The results the model must see, by conversation prefix and call id: rewrites and results it substituted for blocked calls. The harness keeps its own copy of each in its history and re-sends it forever, so every request has these re-applied before the graph is matched — the trace holds the pinned version, and a differing copy would fork it at that node.""" - gated_tools: set[tuple[str, str]] = field(default_factory=set) + gated_tools: set[ToolResultKey] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" + tool_dialects: dict[int, DialectName] = field(default_factory=dict) + """Wire dialect of each issuing node, used when its harness asks the tool gate.""" + tool_issuers: dict[tuple[str, str], set[int]] = field(default_factory=dict) + """Sampled nodes that issued a call ID under a replay-equivalent prefix.""" prepared_users: Counter[str] = field(default_factory=Counter) last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and @@ -239,32 +242,66 @@ def ctx(self) -> ModelContext: def stopped(self) -> bool: return self.trace.stop_condition is not None - def tool_result_keys(self, request: Request) -> dict[int, tuple[str, str]]: - """Identify results by their issuing conversation prefix and call id. Canonical - pinned history gives native and rewritten replays the same occurrence keys.""" + def tool_result_keys( + self, + request: Request, + dialect: DialectName | None = None, + known_path: Sequence[int] = (), + ) -> dict[int, ToolResultKey]: + """Use the sampled node and call ID for a model-issued tool. A harness-supplied + call outside the sampled graph uses its conversation prefix instead.""" prefix = hashlib.blake2b(digest_size=16) - calls: dict[str, tuple[str, str]] = {} - keys: dict[int, tuple[str, str]] = {} + calls: dict[str, ToolResultKey] = {} + keys: dict[int, ToolResultKey] = {} for position, message in enumerate(request.messages): if isinstance(message, ToolMessage) and message.tool_call_id in calls: key = calls[message.tool_call_id] + issuer, call_id = key + if isinstance(issuer, str) and ( + issuers := self.tool_issuers.get((issuer, call_id)) + ): + # A follow-up can change tools independently of its conversation. + if len(issuers) != 1: + raise HarnessError( + f"Ambiguous issuing turn for tool call {call_id!r}" + ) + key = (next(iter(issuers)), call_id) keys[position] = key message = self.pinned_tool_results.get(key, message) - prefix.update(graph.message_hash(message).encode()) - if isinstance(message, AssistantMessage): - calls.update( - (call.id, (prefix.hexdigest(), call.id)) - for call in message.tool_calls or [] - ) + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) + if isinstance(message, AssistantMessage) and message.tool_calls: + prefix_hash = prefix.hexdigest() + for call in message.tool_calls: + alias = (prefix_hash, call.id) + if ( + position < len(known_path) + and self.trace.nodes[known_path[position]].sampled + ): + issuer = known_path[position] + self.tool_issuers.setdefault(alias, set()).add(issuer) + calls[call.id] = (issuer, call.id) + else: + calls[call.id] = alias return keys - def pin_tool_results(self, request: Request) -> Request: + def pin_tool_results( + self, + request: Request, + dialect: DialectName | None = None, + known_path: Sequence[int] = (), + ) -> Request: """Re-impose the pinned results on the harness's request. A result for a blocked call the harness never gated means the call ran anyway: the transcript would no longer describe the sandbox, so that fails the rollout instead of being papered over.""" + if not self.pinned_tool_results: + return request messages = list(request.messages) changed = False - for position, key in self.tool_result_keys(request).items(): + for position, key in self.tool_result_keys( + request, dialect, known_path + ).items(): pinned = self.pinned_tool_results.get(key) if pinned is None: continue @@ -283,14 +320,15 @@ async def rewrite_request( *, run_stops: bool = True, dialect: DialectName | None = None, + known_path: Sequence[int] = (), ) -> tuple[Request, list[InterceptRecord], str | None]: """Run typed request interceptors and stops over one canonical request. Only the uncommitted tail's user and tool messages may change; a rewritten tool result is pinned for the rest of the rollout.""" - request = self.pin_tool_results(request) - tool_keys = self.tool_result_keys(request) + request = self.pin_tool_results(request, dialect, known_path) if not self.request_interceptors and (not run_stops or not self.request_stops): return request, [], None + tool_keys = self.tool_result_keys(request, dialect, known_path) candidates: set[int] = set() tail_start = graph.message_prefix_len(self.trace, request.messages, dialect) if self.request_interceptors: @@ -399,7 +437,7 @@ async def rewrite_request( f"request interception failed: {type(error).__name__}: {error}" ) from error current = request.model_copy(update={"messages": messages}) - tool_keys = self.tool_result_keys(current) + tool_keys = self.tool_result_keys(current, dialect, known_path) for position in candidates: after = current.messages[position] if ( @@ -422,9 +460,16 @@ async def prepare_users( self, request: Request ) -> tuple[Request, list[InterceptRecord]]: """Intercept caller-owned user turns before the harness stores them.""" - branch = self.trace.messages + known_path = ( + graph.path_node_ids(self.trace, len(self.trace.nodes) - 1) + if self.trace.nodes + else [] + ) + branch = [self.trace.nodes[i].message for i in known_path] rewritten, records, _ = await self.rewrite_request( - Request(messages=[*branch, *request.messages]), run_stops=False + Request(messages=[*branch, *request.messages]), + run_stops=False, + known_path=known_path, ) tail = rewritten.messages[len(branch) :] self.prepared_users.update( @@ -489,27 +534,37 @@ async def rewrite_response( ) from error return response, records, None - async def gate_tool_calls(self, node: int) -> str | None: + async def gate_tool_calls( + self, + node: int, + dialect: DialectName | None = None, + ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a `@stop` refuses the whole response and a rewrite becomes that call's verdict — the harness's gate denies it and the model sees the rewritten result instead. Returns the name of the stop that fired, if any.""" assistant = self.trace.nodes[node].message - if not (self.request_interceptors or self.request_stops) or not ( - isinstance(assistant, AssistantMessage) and assistant.tool_calls - ): + if not (self.request_interceptors or self.request_stops): return None - branch = graph.path(self.trace, node) + if dialect is not None: + self.tool_dialects[node] = dialect + if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: + return None + known_path = graph.path_node_ids(self.trace, node) + branch = [self.trace.nodes[i].message for i in known_path] + tools = self.trace.nodes[known_path[0]].tools for call in assistant.tool_calls: probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) - request = Request(messages=[*branch, probe], tools=self.trace.tools or None) - key = self.tool_result_keys(request)[len(branch)] + request = Request(messages=[*branch, probe], tools=tools or None) + key = (node, call.id) if key in self.tool_decisions: raise HarnessError( f"tool call id {call.id!r} was reused in the same assistant turn" ) - request, records, stopped = await self.rewrite_request(request) + request, records, stopped = await self.rewrite_request( + request, dialect=dialect, known_path=known_path + ) self.trace.request_rewrites.extend(records) if stopped is not None: return stopped @@ -575,11 +630,7 @@ async def decide_tool( ] or matches if len(matches) == 1: leaf, call = matches[0] - branch = graph.path(self.trace, leaf) - request = Request( - messages=[*branch, ToolMessage(tool_call_id=call.id, content="")] - ) - key = self.tool_result_keys(request)[len(branch)] + key = (leaf, call.id) self.gated_tools.add(key) verdict = self.tool_decisions.get(key) elif matches or not name or len(leaves) != 1: @@ -588,7 +639,8 @@ async def decide_tool( f"{len(matches)} proposed calls" ) else: - *branch, assistant = graph.path(self.trace, leaves[0][0]) + known_path = graph.path_node_ids(self.trace, leaves[0][0]) + *branch, assistant = [self.trace.nodes[i].message for i in known_path] assert isinstance(assistant, AssistantMessage) call = ToolCall(id=tool_call_id, name=name, arguments=json.dumps(arguments)) assistant = assistant.model_copy( @@ -597,8 +649,11 @@ async def decide_tool( probe = ToolMessage(tool_call_id=call.id, content="", name=name) request, records, stopped = await self.rewrite_request( Request( - messages=[*branch, assistant, probe], tools=self.trace.tools or None - ) + messages=[*branch, assistant, probe], + tools=self.trace.nodes[known_path[0]].tools or None, + ), + dialect=self.tool_dialects.get(leaves[0][0]), + known_path=known_path[:-1], ) self.trace.request_rewrites.extend(records) if stopped is not None: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 6bcbe22179..e695a3aaca 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -742,12 +742,7 @@ def branches(self) -> list[Branch]: } branches: list[Branch] = [] for i, leaf in enumerate(graph.leaves(self)): - path: list[int] = [] - nid: int | None = leaf - while nid is not None: - path.append(nid) - nid = self.nodes[nid].parent - path.reverse() + path = graph.path_node_ids(self, leaf) is_compaction_attempt = any( link.type == "compaction_attempt" for link in self.nodes[leaf].semantic_parents