Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 108 additions & 0 deletions tests/unit/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@
import pytest

import verifiers.v1 as vf
from verifiers.v1 import graph
from verifiers.v1.agent import Interaction
from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute
from verifiers.v1.errors import HarnessError
from verifiers.v1.graph import MessageNode
from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks
from verifiers.v1.rollout import Rollout, RolloutTimeouts
from verifiers.v1.session import RolloutSession
from verifiers.v1.types import AssistantMessage


Expand All @@ -33,6 +38,109 @@ def make_trace() -> vf.Trace:
)


async def test_each_sample_keeps_its_tool_result_across_assistant_replays():
replacements = iter(
["First result", "Second result", "Follow-up result", "Third result"]
)

def replace_result(request: vf.Request) -> vf.Request | None:
tool = request.messages[-1]
if isinstance(tool, vf.ToolMessage) and tool.content == "":
tool.content = next(replacements)
return request
return None

trace = make_trace()
trace._replay_view = lambda message, dialect: (
drop_blank_text_blocks(message) if dialect == "anthropic_messages" else message
)
sampled = AssistantMessage(
content="Ready\n",
text_block_offsets=[5],
tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")],
)
session = RolloutSession(
ModelRoute(ModelContext(model="test", client=EvalClientConfig())),
trace,
request_interceptors=[replace_result],
gates_tools=True,
)
replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None})
other = replay.model_copy(update={"content": "Another action"})
prompt = [vf.UserMessage(content="go")]

def commit(messages, assistant, tools=None):
turn = graph.prepare_turn(
trace, messages, tools=tools, dialect="anthropic_messages"
)
return turn.commit(
vf.Response(
id="", created=0, model="test", message=assistant, finish_reason="stop"
)
)

# The two samples reuse a call ID, but their replayed messages identify them.
catalogs = [[vf.Tool(name=name)] for name in ("filtered_first", "filtered_second")]
for assistant, replayed, expected, tools in [
(sampled, replay, "First result", catalogs[0]),
(other, other, "Second result", catalogs[1]),
]:
node = commit(prompt, assistant)
await session.gate_tool_calls(node, "anthropic_messages")
assert (await session.decide_tool("call"))["action"] == "deny"
request = vf.Request(
messages=[
*prompt,
replayed,
vf.ToolMessage(tool_call_id="call", content="Tool unavailable"),
],
tools=tools,
)
rewritten, _, _ = await session.rewrite_request(
request, dialect="anthropic_messages"
)
assert rewritten.messages[-1].content == expected
if assistant is sampled:
first_request = request
first_path = graph.path_node_ids(trace, node)
commit(rewritten.messages, AssistantMessage(content="done"))

# The next request can change its available tools independently of its history.
changed_catalog = first_request.model_copy(update={"tools": catalogs[1]})
earlier, _, _ = await session.rewrite_request(
changed_catalog, dialect="anthropic_messages"
)
assert earlier.messages[-1].content == "First result"

# An unrelated branch adds a tool to the trace-wide catalog. The next gate
# still knows exactly which sampled ancestors issued its historical calls.
commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")])
follow_up = AssistantMessage(
tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")]
)
node = commit(rewritten.messages, follow_up)
await session.gate_tool_calls(node, "anthropic_messages")
verdict = await session.decide_tool("next")
assert verdict["message"]["content"] == "Follow-up result"

# The third sample reuses the first sample's replay and call ID under a new catalog.
node = commit(prompt, sampled)
await session.gate_tool_calls(node, "anthropic_messages")
assert (await session.decide_tool("call"))["message"]["content"] == "Third result"
with pytest.raises(HarnessError, match="Ambiguous issuing turn"):
await session.rewrite_request(changed_catalog, dialect="anthropic_messages")
with pytest.raises(HarnessError, match="Ambiguous issuing turn"):
await session.rewrite_request(first_request, dialect="anthropic_messages")
with pytest.raises(HarnessError, match="Ambiguous issuing turn"):
session.pin_tool_results(
first_request.model_copy(update={"tools": None}), "anthropic_messages"
)
restored = session.pin_tool_results(
changed_catalog, "anthropic_messages", first_path
)
assert restored.messages[-1].content == "First result"


def make_rollout(trace: vf.Trace, session, time_remaining=None):
run = FailingSegmentRollout()
run.trace = trace
Expand Down
29 changes: 18 additions & 11 deletions verifiers/v1/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,9 @@ def _tools_hash(tools: list[Tool] | None) -> str:
return hashlib.blake2b(data, digest_size=16).hexdigest()


def _message_key(trace: Trace, message: Message, dialect: DialectName | None) -> str:
def replay_message_hash(
trace: Trace, message: Message, dialect: DialectName | None
) -> str:
"""`message_hash` of the message as the trace's harness replays it over `dialect`,
so a turn the harness re-sends with cosmetic rewrites still matches the node it
came from."""
Expand All @@ -433,7 +435,7 @@ def _node_key(
return (
parent,
_tools_hash(tools) if parent is None else None,
_message_key(trace, message, dialect),
replay_message_hash(trace, message, dialect),
)


Expand Down Expand Up @@ -472,12 +474,12 @@ def message_prefix_len(
parents: list[int | None] = [None]
matched = 0
for message in prompt:
key = _message_key(trace, message, dialect)
key = replay_message_hash(trace, message, dialect)
parents = [
node_id
for parent in parents
for node_id in children.get(parent, [])
if _message_key(trace, trace.nodes[node_id].message, dialect) == key
if replay_message_hash(trace, trace.nodes[node_id].message, dialect) == key
]
if not parents:
break
Expand Down Expand Up @@ -743,7 +745,7 @@ def prepare_turn(
if len(children) == 1 and trace.nodes[children[0]].message == msg:
existing = children[0]
if existing is None:
message_key = _message_key(trace, msg, dialect)
message_key = replay_message_hash(trace, msg, dialect)
existing = idx.get(
(parent, tools_hash if parent is None else None, message_key)
)
Expand Down Expand Up @@ -1140,12 +1142,17 @@ def leaves(trace: Trace) -> list[int]:
return [i for i in range(len(trace.nodes)) if i not in has_child]


def path(trace: Trace, node: int) -> list[Message]:
"""The conversation ending at `node`: its root-to-node messages."""
messages: list[Message] = []
def path_node_ids(trace: Trace, node: int) -> list[int]:
"""The physical ancestry from the root through the given node."""
nodes: list[int] = []
current: int | None = node
while current is not None:
messages.append(trace.nodes[current].message)
nodes.append(current)
current = trace.nodes[current].parent
messages.reverse()
return messages
nodes.reverse()
return nodes


def path(trace: Trace, node: int) -> list[Message]:
"""The conversation ending at `node`: its root-to-node messages."""
return [trace.nodes[i].message for i in path_node_ids(trace, node)]
2 changes: 1 addition & 1 deletion verifiers/v1/interception/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1013,7 +1013,7 @@ async def sample() -> web.Response:
session.consume_prepared(turn.tail)
session.trace.response_rewrites.extend(response_rewrites)
if stopped is None:
stopped = await session.gate_tool_calls(node)
stopped = await session.gate_tool_calls(node, dialect.name)
if stopped is not None:
session.trace.stop(stopped)
return _error_response(
Expand Down
Loading
Loading