Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tests/integration/fixtures/echo_agentic.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ def load(self) -> list[EchoAgenticTask]:
idx=0,
prompt=(
f"Use the bash tool to write exactly the text '{phrase}' to a file "
f"named {TARGET} in the current directory, then finish."
f"named {TARGET} in the current directory. Use only the arguments "
"required to run the command, then finish."
),
system_prompt=SYSTEM,
answer=phrase,
Expand Down
6 changes: 3 additions & 3 deletions tests/integration/fixtures/network_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ def load(self) -> list[NetworkPolicyTask]:
"allow": {"network": vf.NetworkPolicyConfig(allow=[ALLOWED])},
}
prompt = (
f"Your only task: run the command `python3 {PROBE_PATH}` once in your "
"shell, then reply with exactly what it printed. Any network policy is "
"already enforced, so do not inspect, configure, or change the network or "
f"Use your shell tool now to run `python3 {PROBE_PATH}` exactly once. "
"Wait for the command to finish, then reply with exactly what it printed. "
"The network policy is already enforced, so do not inspect or change the network or "
"firewall, and do not run any other command."
)
return [
Expand Down
23 changes: 20 additions & 3 deletions tests/integration/grid.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
"mini-swe-agent": "https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/models/litellm_model.py#L79-L90",
# Drops historical runtime-context messages and can relocate the current
# prompt's context around user turns.
"openclaw": "https://github.com/openclaw/openclaw/blob/v2026.10.9/src/agents/internal-runtime-context.ts#L403-L513",
"openclaw": "https://github.com/openclaw/openclaw/blob/v2026.11.5/src/agents/internal-runtime-context.ts#L445-L563",
}


Expand All @@ -94,8 +94,25 @@ def assert_one_branch(
assert set(forks) <= FORKS.keys(), (
f"no FORKS reason for {set(forks) - FORKS.keys()}"
)
if harness.id not in forks:
assert trace.num_branches == 1, f"{trace.num_branches} branches"
if harness.id in forks or trace.num_branches == 1:
return
first_child: dict[int | None, int] = {}
for i, node in enumerate(trace.nodes):
first = first_child.setdefault(node.parent, i)
if first != i:
sibling = trace.nodes[first]
a, b = sibling.message, node.message
fields = type(a).model_fields.keys() | type(b).model_fields.keys()
changed = sorted(
k for k in fields if getattr(a, k, None) != getattr(b, k, None)
)
if sibling.tools != node.tools:
changed.append("tools")
pytest.fail(
f"{trace.num_branches} branches at parent {node.parent}, "
f"nodes {first} ({a.role}) and {i} ({b.role}), differing fields {changed}"
)
pytest.fail(f"{trace.num_branches} branches")


def agent_seats(taskset: str, env_id: str = "") -> list[str]:
Expand Down
130 changes: 130 additions & 0 deletions tests/unit/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,16 @@
import pytest

import verifiers.v1 as vf
from verifiers.v1 import graph
from verifiers.v1.agent import Interaction
from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute
from verifiers.v1.configs.runtime import NetworkPolicyConfig
from verifiers.v1.dialects.anthropic import AnthropicDialect
from verifiers.v1.errors import HarnessError
from verifiers.v1.graph import MessageNode
from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks
from verifiers.v1.rollout import Rollout, RolloutTimeouts
from verifiers.v1.session import RolloutSession
from verifiers.v1.types import AssistantMessage


Expand All @@ -33,6 +40,129 @@ def make_trace() -> vf.Trace:
)


async def test_each_sample_keeps_its_tool_result_across_assistant_replays():
replacements = iter(
["First result", "Second result", "Follow-up result", "Third result"]
)

def replace_result(request: vf.Request) -> vf.Request | None:
tool = request.messages[-1]
if isinstance(tool, vf.ToolMessage) and tool.content == "":
tool.content = next(replacements)
return request
return None

trace = make_trace()
trace._replay_view = lambda message, dialect: (
drop_blank_text_blocks(message) if dialect == "anthropic_messages" else message
)
sampled = AssistantMessage(
content="Ready\n",
text_block_offsets=[5],
tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")],
)
session = RolloutSession(
ModelRoute(ModelContext(model="test", client=EvalClientConfig())),
trace,
request_interceptors=[replace_result],
gates_tools=True,
)
replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None})
prompt = [vf.UserMessage(content="go")]
dialect = AnthropicDialect()

def restrict(provider_tool: str):
body = {
"messages": [{"role": "user", "content": "go"}],
"tools": [
{"name": "lookup", "input_schema": {"type": "object"}},
{"name": provider_tool.rsplit("_", 1)[0], "type": provider_tool},
],
}
native = dialect.parse_request(body)
filtered, _ = dialect.mediate_external_capabilities(
body, NetworkPolicyConfig(allow=[])
)
return native, dialect.parse_request(filtered)

def commit(messages, assistant, tools=None):
turn = graph.prepare_turn(
trace, messages, tools=tools, dialect="anthropic_messages"
)
return turn.commit(
vf.Response(
id="", created=0, model="test", message=assistant, finish_reason="stop"
)
)

# Removing a provider tool also adds a notice to the model-facing prompt.
first_native, filtered = restrict("web_search_20250305")
second_native, second_filtered = restrict("web_fetch_20250910")
assert filtered == second_filtered
assert [tool.name for tool in filtered.tools or []] == ["lookup"]
assert filtered.messages != prompt
for assistant, expected, native in [
(sampled, "First result", first_native),
(replay, "Second result", second_native),
]:
node = commit(filtered.messages, assistant, filtered.tools)
await session.gate_tool_calls(node, "anthropic_messages", native)
assert (await session.decide_tool("call"))["action"] == "deny"
request = vf.Request(
messages=[
*prompt,
replay,
vf.ToolMessage(tool_call_id="call", content="Tool unavailable"),
],
tools=native.tools,
)
rewritten, _, _ = await session.rewrite_request(
request, dialect="anthropic_messages"
)
assert rewritten.messages[-1].content == expected
model_history = [*filtered.messages, *rewritten.messages[len(prompt) :]]
commit(model_history, AssistantMessage(content="done"), filtered.tools)

first_request = request.model_copy(update={"tools": first_native.tools})
earlier, _, _ = await session.rewrite_request(
first_request, dialect="anthropic_messages"
)
assert earlier.messages[-1].content == "First result"

# An unrelated branch adds a tool to the trace-wide catalog. The next gate
# still knows exactly which sampled ancestors issued its historical calls.
commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")])
follow_up = AssistantMessage(
tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")]
)
node = commit(model_history, follow_up, filtered.tools)
await session.gate_tool_calls(node, "anthropic_messages", rewritten)
verdict = await session.decide_tool("next")
assert verdict["message"]["content"] == "Follow-up result"
continued = request.model_copy(
update={
"messages": [
*request.messages,
follow_up,
vf.ToolMessage(tool_call_id="next", content="Tool unavailable"),
]
}
)
final, _, _ = await session.rewrite_request(continued, dialect="anthropic_messages")
assert [m.content for m in final.messages if isinstance(m, vf.ToolMessage)] == [
"Second result",
"Follow-up result",
]

# Even the catalog is identical this time. A bare receipt cannot identify which
# of two different substitutions belongs to it.
node = commit(filtered.messages, sampled, filtered.tools)
await session.gate_tool_calls(node, "anthropic_messages", first_native)
assert (await session.decide_tool("call"))["message"]["content"] == "Third result"
with pytest.raises(HarnessError, match="Ambiguous issuing turn"):
await session.rewrite_request(first_request, dialect="anthropic_messages")


def make_rollout(trace: vf.Trace, session, time_remaining=None):
run = FailingSegmentRollout()
run.trace = trace
Expand Down
47 changes: 47 additions & 0 deletions tests/unit/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@

import verifiers.v1 as vf
from verifiers.v1 import graph
from verifiers.v1.dialects.anthropic import AnthropicDialect
from verifiers.v1.harnesses.claude_code.harness import (
ClaudeCodeHarness,
ClaudeCodeHarnessConfig,
)
from verifiers.v1.harnesses.utils.replay import drop_blank_text, strip_text
from verifiers.v1.types import TurnTokens

Expand Down Expand Up @@ -321,6 +326,48 @@ def test_replay_view_matches_cosmetically_rewritten_turns(view, sampled, replaye
assert trace.num_branches == 1


@pytest.mark.parametrize(
"blocks,branches",
[(["One ", "\n\n", " two"], 1), (["One \n\n two"], 2)],
)
def test_claude_replay_only_drops_independently_blank_text_blocks(blocks, branches):
dialect = AnthropicDialect()
harness = ClaudeCodeHarness(ClaudeCodeHarnessConfig())
response = dialect.validate_response(
{
"type": "message",
"id": "msg",
"model": "test",
"role": "assistant",
"content": [{"type": "text", "text": block} for block in blocks],
"usage": {"input_tokens": 1, "output_tokens": 1},
}
)
sampled = dialect.parse_response(response).message
assert sampled.content == "One \n\n two"
(replayed,) = dialect.parse_request(
{
"messages": [
{
"role": "assistant",
"content": [
{"type": "text", "text": "One "},
{"type": "text", "text": " two"},
],
}
]
}
).messages
fields = {"content", "text_block_offsets"}
trace = _replayed_trace(
harness.replay_view,
sampled.model_dump(include=fields),
replayed.model_dump(include=fields),
dialect="anthropic_messages",
)
assert trace.num_branches == branches


@pytest.mark.parametrize(
"dialect,branches", [("chat_completions", 1), ("responses", 2)]
)
Expand Down
2 changes: 2 additions & 0 deletions verifiers/v1/acp/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ class ACPConfig:
system_prompt: str | None = None
session_meta: JsonObject | None = None
client_capabilities: JsonObject | None = None
session_title: str | None = None


class ACPHarness(Harness[ConfigT]):
Expand Down Expand Up @@ -335,6 +336,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult:
"system_prompt": self.config.system_prompt or "",
"session_meta": self.config.session_meta or {},
"client_capabilities": self.config.client_capabilities or {},
"session_title": self.config.session_title,
"tool_interception": {
"url": self.tool_interception_url,
"secret": self.secret,
Expand Down
4 changes: 4 additions & 0 deletions verifiers/v1/acp/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,10 @@ async def start(self, config: dict) -> None:
mcp_servers=mcp_servers(config),
**config["session_meta"],
)
if title := config.get("session_title"):
await self.connection.ext_method(
"session/rename", {"sessionId": session.session_id, "title": title}
)
except BaseException:
with suppress(BaseException):
await self.stack.aclose()
Expand Down
11 changes: 9 additions & 2 deletions verifiers/v1/dialects/anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import re
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from itertools import accumulate

from anthropic.types import Message as AnthropicMessage
from anthropic.types import Usage as AnthropicUsage
Expand Down Expand Up @@ -241,6 +242,10 @@ def content_to_wire(content) -> str | list[dict]:
return blocks


def _text_block_offsets(blocks: list[str]) -> list[int] | None:
return list(accumulate(map(len, blocks[:-1]))) or None


def parse_messages(body: dict) -> Messages:
"""The request's top-level `system` + `messages` -> typed messages. Assistant turns fold
their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's
Expand All @@ -260,7 +265,7 @@ def parse_messages(body: dict) -> Messages:
block for block in blocks if block["type"] not in ("text", "tool_use")
]
state.sort(key=lambda block: THINKING_ORDER.get(block["type"], 2))
text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text")
text = [b.get("text", "") for b in blocks if b.get("type") == "text"]
reasoning = "".join(
b.get("thinking", "") for b in blocks if b.get("type") == "thinking"
)
Expand All @@ -276,7 +281,8 @@ def parse_messages(body: dict) -> Messages:
]
prompt.append(
AssistantMessage(
content=text or None,
content="".join(text) or None,
text_block_offsets=_text_block_offsets(text),
reasoning_content=reasoning or None,
tool_calls=calls or None,
provider_state=state or None,
Expand Down Expand Up @@ -351,6 +357,7 @@ def response_from_wire(message: AnthropicMessage) -> Response:
model=message.model,
message=AssistantMessage(
content="".join(content) or None,
text_block_offsets=_text_block_offsets(content),
reasoning_content="".join(reasoning) or None,
tool_calls=calls or None,
provider_state=state or None,
Expand Down
Loading