diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index 9424ac67c2..163ab92b0f 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -10,6 +10,8 @@ from verifiers.v1 import graph from verifiers.v1.agent import Interaction from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute +from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.dialects.anthropic import AnthropicDialect from verifiers.v1.errors import HarnessError from verifiers.v1.graph import MessageNode from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks @@ -68,6 +70,21 @@ def replace_result(request: vf.Request) -> vf.Request | None: replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) other = replay.model_copy(update={"content": "Another action"}) prompt = [vf.UserMessage(content="go")] + dialect = AnthropicDialect() + + def restrict(provider_tool: str): + body = { + "messages": [{"role": "user", "content": "go"}], + "tools": [ + {"name": "lookup", "input_schema": {"type": "object"}}, + {"name": provider_tool.rsplit("_", 1)[0], "type": provider_tool}, + ], + } + native = dialect.parse_request(body) + filtered, _ = dialect.mediate_external_capabilities( + body, NetworkPolicyConfig(allow=[]) + ) + return native, dialect.parse_request(filtered) def commit(messages, assistant, tools=None): turn = graph.prepare_turn( @@ -79,14 +96,18 @@ def commit(messages, assistant, tools=None): ) ) - # The two samples reuse a call ID, but their replayed messages identify them. - catalogs = [[vf.Tool(name=name)] for name in ("filtered_first", "filtered_second")] - for assistant, replayed, expected, tools in [ - (sampled, replay, "First result", catalogs[0]), - (other, other, "Second result", catalogs[1]), + # Removing a provider tool also adds a notice to the model-facing prompt. + first_native, filtered = restrict("web_search_20250305") + second_native, second_filtered = restrict("web_fetch_20250910") + assert filtered == second_filtered + assert [tool.name for tool in filtered.tools or []] == ["lookup"] + assert filtered.messages != prompt + for assistant, replayed, expected, native in [ + (sampled, replay, "First result", first_native), + (other, other, "Second result", second_native), ]: - node = commit(prompt, assistant) - await session.gate_tool_calls(node, "anthropic_messages") + node = commit(filtered.messages, assistant, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", native.messages) assert (await session.decide_tool("call"))["action"] == "deny" request = vf.Request( messages=[ @@ -94,7 +115,7 @@ def commit(messages, assistant, tools=None): replayed, vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), ], - tools=tools, + tools=native.tools, ) rewritten, _, _ = await session.rewrite_request( request, dialect="anthropic_messages" @@ -103,10 +124,19 @@ def commit(messages, assistant, tools=None): if assistant is sampled: first_request = request first_path = graph.path_node_ids(trace, node) - commit(rewritten.messages, AssistantMessage(content="done")) + model_history = [*filtered.messages, *rewritten.messages[len(prompt) :]] + model_receipt = vf.Request( + messages=[*filtered.messages, replayed, request.messages[-1]], + tools=filtered.tools, + ) + restored = session.pin_tool_results(model_receipt, "anthropic_messages") + assert restored.messages[-1].content == expected + if native is first_native: + first_model_receipt = model_receipt + commit(model_history, AssistantMessage(content="done"), filtered.tools) # The next request can change its available tools independently of its history. - changed_catalog = first_request.model_copy(update={"tools": catalogs[1]}) + changed_catalog = first_request.model_copy(update={"tools": second_native.tools}) earlier, _, _ = await session.rewrite_request( changed_catalog, dialect="anthropic_messages" ) @@ -118,14 +148,28 @@ def commit(messages, assistant, tools=None): follow_up = AssistantMessage( tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")] ) - node = commit(rewritten.messages, follow_up) - await session.gate_tool_calls(node, "anthropic_messages") + node = commit(model_history, follow_up, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", rewritten.messages) verdict = await session.decide_tool("next") assert verdict["message"]["content"] == "Follow-up result" + continued = request.model_copy( + update={ + "messages": [ + *request.messages, + follow_up, + vf.ToolMessage(tool_call_id="next", content="Tool unavailable"), + ] + } + ) + final, _, _ = await session.rewrite_request(continued, dialect="anthropic_messages") + assert [m.content for m in final.messages if isinstance(m, vf.ToolMessage)] == [ + "Second result", + "Follow-up result", + ] # The third sample reuses the first sample's replay and call ID under a new catalog. - node = commit(prompt, sampled) - await session.gate_tool_calls(node, "anthropic_messages") + node = commit(filtered.messages, sampled, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", second_native.messages) assert (await session.decide_tool("call"))["message"]["content"] == "Third result" with pytest.raises(HarnessError, match="Ambiguous issuing turn"): await session.rewrite_request(changed_catalog, dialect="anthropic_messages") @@ -135,6 +179,8 @@ def commit(messages, assistant, tools=None): session.pin_tool_results( first_request.model_copy(update={"tools": None}), "anthropic_messages" ) + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + session.pin_tool_results(first_model_receipt, "anthropic_messages") restored = session.pin_tool_results( changed_catalog, "anthropic_messages", first_path ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 9f44e9550e..b442535afb 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -67,7 +67,7 @@ from verifiers.v1.semantic import ACPInfo, extract_acp_info, routing_session_id from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan -from verifiers.v1.types import FinishReason, Response, Usage +from verifiers.v1.types import FinishReason, Response, ToolMessage, Usage from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.format import format_time from verifiers.v1.utils.logging import clip, get_logger, isolated @@ -881,6 +881,7 @@ async def coalesced( dialect, f"rollout stopped: {refused}", status=400 ) original_request = model_request + harness_history = original_request.messages model_request, request_rewrites, stopped = await session.rewrite_request( model_request, dialect=dialect.name ) @@ -888,6 +889,14 @@ async def coalesced( # A pinned tool result changes the request without a fresh record. if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) + # Match future receipts against the prompt the harness will replay. + # It retains its own user messages. We reapply pinned tool results. + harness_history = [ + after if isinstance(after, ToolMessage) else before + for before, after in zip( + original_request.messages, model_request.messages, strict=True + ) + ] except RolloutError as error: return self._fail(session, dialect, error) except Exception as error: # noqa: BLE001 - surface task hook failures @@ -1013,7 +1022,9 @@ async def sample() -> web.Response: session.consume_prepared(turn.tail) session.trace.response_rewrites.extend(response_rewrites) if stopped is None: - stopped = await session.gate_tool_calls(node, dialect.name) + stopped = await session.gate_tool_calls( + node, dialect.name, harness_history + ) if stopped is not None: session.trace.stop(stopped) return _error_response( diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index d25971f297..c9f32f4a0c 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -280,7 +280,6 @@ def tool_result_keys( and self.trace.nodes[known_path[position]].sampled ): issuer = known_path[position] - self.tool_issuers.setdefault(alias, set()).add(issuer) calls[call.id] = (issuer, call.id) else: calls[call.id] = alias @@ -538,6 +537,7 @@ async def gate_tool_calls( self, node: int, dialect: DialectName | None = None, + harness_history: Messages | None = None, ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a @@ -554,6 +554,20 @@ async def gate_tool_calls( known_path = graph.path_node_ids(self.trace, node) branch = [self.trace.nodes[i].message for i in known_path] tools = self.trace.nodes[known_path[0]].tools + # Live harnesses replay their own input. A resumed harness may replay the + # model-facing transcript, including any policy notices or rewritten user turns. + histories = [branch] + if harness_history is not None and harness_history != branch[:-1]: + histories.append([*harness_history, assistant]) + for history in histories: + prefix = hashlib.blake2b(digest_size=16) + for message in history: + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) + prefix_hash = prefix.hexdigest() + for call in assistant.tool_calls: + self.tool_issuers.setdefault((prefix_hash, call.id), set()).add(node) for call in assistant.tool_calls: probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) request = Request(messages=[*branch, probe], tools=tools or None)