diff --git a/tests/integration/fixtures/echo_agentic.py b/tests/integration/fixtures/echo_agentic.py index a0402ec63a..35ebc057cf 100644 --- a/tests/integration/fixtures/echo_agentic.py +++ b/tests/integration/fixtures/echo_agentic.py @@ -53,7 +53,8 @@ def load(self) -> list[EchoAgenticTask]: idx=0, prompt=( f"Use the bash tool to write exactly the text '{phrase}' to a file " - f"named {TARGET} in the current directory, then finish." + f"named {TARGET} in the current directory. Use only the arguments " + "required to run the command, then finish." ), system_prompt=SYSTEM, answer=phrase, diff --git a/tests/integration/fixtures/network_policy.py b/tests/integration/fixtures/network_policy.py index 8e6489cce8..47e86a7b7f 100644 --- a/tests/integration/fixtures/network_policy.py +++ b/tests/integration/fixtures/network_policy.py @@ -68,9 +68,9 @@ def load(self) -> list[NetworkPolicyTask]: "allow": {"network": vf.NetworkPolicyConfig(allow=[ALLOWED])}, } prompt = ( - f"Your only task: run the command `python3 {PROBE_PATH}` once in your " - "shell, then reply with exactly what it printed. Any network policy is " - "already enforced, so do not inspect, configure, or change the network or " + f"Use your shell tool now to run `python3 {PROBE_PATH}` exactly once. " + "Wait for the command to finish, then reply with exactly what it printed. " + "The network policy is already enforced, so do not inspect or change the network or " "firewall, and do not run any other command." ) return [ diff --git a/tests/integration/grid.py b/tests/integration/grid.py index b93a443bfe..d7ace173e1 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -83,7 +83,7 @@ "mini-swe-agent": "https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/models/litellm_model.py#L79-L90", # Drops historical runtime-context messages and can relocate the current # prompt's context around user turns. - "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.10.9/src/agents/internal-runtime-context.ts#L403-L513", + "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.11.5/src/agents/internal-runtime-context.ts#L445-L563", } @@ -94,8 +94,25 @@ def assert_one_branch( assert set(forks) <= FORKS.keys(), ( f"no FORKS reason for {set(forks) - FORKS.keys()}" ) - if harness.id not in forks: - assert trace.num_branches == 1, f"{trace.num_branches} branches" + if harness.id in forks or trace.num_branches == 1: + return + first_child: dict[int | None, int] = {} + for i, node in enumerate(trace.nodes): + first = first_child.setdefault(node.parent, i) + if first != i: + sibling = trace.nodes[first] + a, b = sibling.message, node.message + fields = type(a).model_fields.keys() | type(b).model_fields.keys() + changed = sorted( + k for k in fields if getattr(a, k, None) != getattr(b, k, None) + ) + if sibling.tools != node.tools: + changed.append("tools") + pytest.fail( + f"{trace.num_branches} branches at parent {node.parent}, " + f"nodes {first} ({a.role}) and {i} ({b.role}), differing fields {changed}" + ) + pytest.fail(f"{trace.num_branches} branches") def agent_seats(taskset: str, env_id: str = "") -> list[str]: diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..ada7779824 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -7,9 +7,16 @@ import pytest import verifiers.v1 as vf +from verifiers.v1 import graph from verifiers.v1.agent import Interaction +from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute +from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.dialects.anthropic import AnthropicDialect +from verifiers.v1.errors import HarnessError from verifiers.v1.graph import MessageNode +from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.rollout import Rollout, RolloutTimeouts +from verifiers.v1.session import RolloutSession from verifiers.v1.types import AssistantMessage @@ -33,6 +40,129 @@ def make_trace() -> vf.Trace: ) +async def test_each_sample_keeps_its_tool_result_across_assistant_replays(): + replacements = iter( + ["First result", "Second result", "Follow-up result", "Third result"] + ) + + def replace_result(request: vf.Request) -> vf.Request | None: + tool = request.messages[-1] + if isinstance(tool, vf.ToolMessage) and tool.content == "": + tool.content = next(replacements) + return request + return None + + trace = make_trace() + trace._replay_view = lambda message, dialect: ( + drop_blank_text_blocks(message) if dialect == "anthropic_messages" else message + ) + sampled = AssistantMessage( + content="Ready\n", + text_block_offsets=[5], + tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")], + ) + session = RolloutSession( + ModelRoute(ModelContext(model="test", client=EvalClientConfig())), + trace, + request_interceptors=[replace_result], + gates_tools=True, + ) + replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) + prompt = [vf.UserMessage(content="go")] + dialect = AnthropicDialect() + + def restrict(provider_tool: str): + body = { + "messages": [{"role": "user", "content": "go"}], + "tools": [ + {"name": "lookup", "input_schema": {"type": "object"}}, + {"name": provider_tool.rsplit("_", 1)[0], "type": provider_tool}, + ], + } + native = dialect.parse_request(body) + filtered, _ = dialect.mediate_external_capabilities( + body, NetworkPolicyConfig(allow=[]) + ) + return native, dialect.parse_request(filtered) + + def commit(messages, assistant, tools=None): + turn = graph.prepare_turn( + trace, messages, tools=tools, dialect="anthropic_messages" + ) + return turn.commit( + vf.Response( + id="", created=0, model="test", message=assistant, finish_reason="stop" + ) + ) + + # Removing a provider tool also adds a notice to the model-facing prompt. + first_native, filtered = restrict("web_search_20250305") + second_native, second_filtered = restrict("web_fetch_20250910") + assert filtered == second_filtered + assert [tool.name for tool in filtered.tools or []] == ["lookup"] + assert filtered.messages != prompt + for assistant, expected, native in [ + (sampled, "First result", first_native), + (replay, "Second result", second_native), + ]: + node = commit(filtered.messages, assistant, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", native) + assert (await session.decide_tool("call"))["action"] == "deny" + request = vf.Request( + messages=[ + *prompt, + replay, + vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), + ], + tools=native.tools, + ) + rewritten, _, _ = await session.rewrite_request( + request, dialect="anthropic_messages" + ) + assert rewritten.messages[-1].content == expected + model_history = [*filtered.messages, *rewritten.messages[len(prompt) :]] + commit(model_history, AssistantMessage(content="done"), filtered.tools) + + first_request = request.model_copy(update={"tools": first_native.tools}) + earlier, _, _ = await session.rewrite_request( + first_request, dialect="anthropic_messages" + ) + assert earlier.messages[-1].content == "First result" + + # An unrelated branch adds a tool to the trace-wide catalog. The next gate + # still knows exactly which sampled ancestors issued its historical calls. + commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")]) + follow_up = AssistantMessage( + tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")] + ) + node = commit(model_history, follow_up, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", rewritten) + verdict = await session.decide_tool("next") + assert verdict["message"]["content"] == "Follow-up result" + continued = request.model_copy( + update={ + "messages": [ + *request.messages, + follow_up, + vf.ToolMessage(tool_call_id="next", content="Tool unavailable"), + ] + } + ) + final, _, _ = await session.rewrite_request(continued, dialect="anthropic_messages") + assert [m.content for m in final.messages if isinstance(m, vf.ToolMessage)] == [ + "Second result", + "Follow-up result", + ] + + # Even the catalog is identical this time. A bare receipt cannot identify which + # of two different substitutions belongs to it. + node = commit(filtered.messages, sampled, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", first_native) + assert (await session.decide_tool("call"))["message"]["content"] == "Third result" + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + await session.rewrite_request(first_request, dialect="anthropic_messages") + + def make_rollout(trace: vf.Trace, session, time_remaining=None): run = FailingSegmentRollout() run.trace = trace diff --git a/tests/unit/test_graph.py b/tests/unit/test_graph.py index 4df44699ed..92c380e5ab 100644 --- a/tests/unit/test_graph.py +++ b/tests/unit/test_graph.py @@ -5,6 +5,11 @@ import verifiers.v1 as vf from verifiers.v1 import graph +from verifiers.v1.dialects.anthropic import AnthropicDialect +from verifiers.v1.harnesses.claude_code.harness import ( + ClaudeCodeHarness, + ClaudeCodeHarnessConfig, +) from verifiers.v1.harnesses.utils.replay import drop_blank_text, strip_text from verifiers.v1.types import TurnTokens @@ -321,6 +326,48 @@ def test_replay_view_matches_cosmetically_rewritten_turns(view, sampled, replaye assert trace.num_branches == 1 +@pytest.mark.parametrize( + "blocks,branches", + [(["One ", "\n\n", " two"], 1), (["One \n\n two"], 2)], +) +def test_claude_replay_only_drops_independently_blank_text_blocks(blocks, branches): + dialect = AnthropicDialect() + harness = ClaudeCodeHarness(ClaudeCodeHarnessConfig()) + response = dialect.validate_response( + { + "type": "message", + "id": "msg", + "model": "test", + "role": "assistant", + "content": [{"type": "text", "text": block} for block in blocks], + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + sampled = dialect.parse_response(response).message + assert sampled.content == "One \n\n two" + (replayed,) = dialect.parse_request( + { + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "One "}, + {"type": "text", "text": " two"}, + ], + } + ] + } + ).messages + fields = {"content", "text_block_offsets"} + trace = _replayed_trace( + harness.replay_view, + sampled.model_dump(include=fields), + replayed.model_dump(include=fields), + dialect="anthropic_messages", + ) + assert trace.num_branches == branches + + @pytest.mark.parametrize( "dialect,branches", [("chat_completions", 1), ("responses", 2)] ) diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 8f35eb6551..eed94416ff 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -65,6 +65,7 @@ class ACPConfig: system_prompt: str | None = None session_meta: JsonObject | None = None client_capabilities: JsonObject | None = None + session_title: str | None = None class ACPHarness(Harness[ConfigT]): @@ -335,6 +336,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult: "system_prompt": self.config.system_prompt or "", "session_meta": self.config.session_meta or {}, "client_capabilities": self.config.client_capabilities or {}, + "session_title": self.config.session_title, "tool_interception": { "url": self.tool_interception_url, "secret": self.secret, diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index 021729f19c..997b39bbce 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -272,6 +272,10 @@ async def start(self, config: dict) -> None: mcp_servers=mcp_servers(config), **config["session_meta"], ) + if title := config.get("session_title"): + await self.connection.ext_method( + "session/rename", {"sessionId": session.session_id, "title": title} + ) except BaseException: with suppress(BaseException): await self.stack.aclose() diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 0939f00736..520a339e7c 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -10,6 +10,7 @@ import re from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from itertools import accumulate from anthropic.types import Message as AnthropicMessage from anthropic.types import Usage as AnthropicUsage @@ -241,6 +242,10 @@ def content_to_wire(content) -> str | list[dict]: return blocks +def _text_block_offsets(blocks: list[str]) -> list[int] | None: + return list(accumulate(map(len, blocks[:-1]))) or None + + def parse_messages(body: dict) -> Messages: """The request's top-level `system` + `messages` -> typed messages. Assistant turns fold their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's @@ -260,7 +265,7 @@ def parse_messages(body: dict) -> Messages: block for block in blocks if block["type"] not in ("text", "tool_use") ] state.sort(key=lambda block: THINKING_ORDER.get(block["type"], 2)) - text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text") + text = [b.get("text", "") for b in blocks if b.get("type") == "text"] reasoning = "".join( b.get("thinking", "") for b in blocks if b.get("type") == "thinking" ) @@ -276,7 +281,8 @@ def parse_messages(body: dict) -> Messages: ] prompt.append( AssistantMessage( - content=text or None, + content="".join(text) or None, + text_block_offsets=_text_block_offsets(text), reasoning_content=reasoning or None, tool_calls=calls or None, provider_state=state or None, @@ -351,6 +357,7 @@ def response_from_wire(message: AnthropicMessage) -> Response: model=message.model, message=AssistantMessage( content="".join(content) or None, + text_block_offsets=_text_block_offsets(content), reasoning_content="".join(reasoning) or None, tool_calls=calls or None, provider_state=state or None, diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index e47fd68e0c..1e0add3817 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -408,13 +408,15 @@ def add(value: str) -> None: return digest.hexdigest() -def _tools_hash(tools: list[Tool] | None) -> str: +def tool_catalog_hash(tools: list[Tool] | None) -> str: payload = [tool.model_dump(mode="json", exclude_none=True) for tool in tools or []] data = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() return hashlib.blake2b(data, digest_size=16).hexdigest() -def _message_key(trace: Trace, message: Message, dialect: DialectName | None) -> str: +def replay_message_hash( + trace: Trace, message: Message, dialect: DialectName | None +) -> str: """`message_hash` of the message as the trace's harness replays it over `dialect`, so a turn the harness re-sends with cosmetic rewrites still matches the node it came from.""" @@ -432,8 +434,8 @@ def _node_key( ) -> tuple[int | None, str | None, str]: return ( parent, - _tools_hash(tools) if parent is None else None, - _message_key(trace, message, dialect), + tool_catalog_hash(tools) if parent is None else None, + replay_message_hash(trace, message, dialect), ) @@ -472,12 +474,12 @@ def message_prefix_len( parents: list[int | None] = [None] matched = 0 for message in prompt: - key = _message_key(trace, message, dialect) + key = replay_message_hash(trace, message, dialect) parents = [ node_id for parent in parents for node_id in children.get(parent, []) - if _message_key(trace, trace.nodes[node_id].message, dialect) == key + if replay_message_hash(trace, trace.nodes[node_id].message, dialect) == key ] if not parents: break @@ -721,7 +723,7 @@ def prepare_turn( is the request's wire format, whose replay view matches the prompt to stored nodes.""" idx = _head_index(trace, dialect) tools = list(tools or []) - tools_hash = _tools_hash(tools) + tools_hash = tool_catalog_hash(tools) parent: int | None = None path_len = 0 prefix_node_ids: list[int] = [] @@ -743,7 +745,7 @@ def prepare_turn( if len(children) == 1 and trace.nodes[children[0]].message == msg: existing = children[0] if existing is None: - message_key = _message_key(trace, msg, dialect) + message_key = replay_message_hash(trace, msg, dialect) existing = idx.get( (parent, tools_hash if parent is None else None, message_key) ) @@ -1140,12 +1142,17 @@ def leaves(trace: Trace) -> list[int]: return [i for i in range(len(trace.nodes)) if i not in has_child] -def path(trace: Trace, node: int) -> list[Message]: - """The conversation ending at `node`: its root-to-node messages.""" - messages: list[Message] = [] +def path_node_ids(trace: Trace, node: int) -> list[int]: + """The physical ancestry from the root through the given node.""" + nodes: list[int] = [] current: int | None = node while current is not None: - messages.append(trace.nodes[current].message) + nodes.append(current) current = trace.nodes[current].parent - messages.reverse() - return messages + nodes.reverse() + return nodes + + +def path(trace: Trace, node: int) -> list[Message]: + """The conversation ending at `node`: its root-to-node messages.""" + return [trace.nodes[i].message for i in path_node_ids(trace, node)] diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 35451a0449..202d876e2a 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -8,7 +8,7 @@ from verifiers.v1.configs.harness import DialectName, HarnessConfig, PinnedVersion from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir -from verifiers.v1.harnesses.utils.replay import drop_blank_text +from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -16,10 +16,12 @@ CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp-{version}-{acp_version}" PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages" -ACP_VERSION = "0.99.1" +ACP_VERSION = "0.108.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" CLAUDE_CONFIG_ROOT = ".vf-claude" +# Manual mode routes tool approvals to ACP. Auto needs a separate classifier. +CLAUDE_PERMISSIONS = {"defaultMode": "default"} ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -34,7 +36,7 @@ class ClaudeCodeHarnessConfig(HarnessConfig): dialect: Literal["anthropic_messages"] = "anthropic_messages" - version: PinnedVersion = "2.1.314" + version: PinnedVersion = "2.1.327" """Claude Code release to install, pinned for reproducibility.""" @@ -44,10 +46,9 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): SUPPORTS_SKILLS = True SUPPORTS_TOOL_INTERCEPTION = True - # Claude Code drops whitespace-only text blocks on replay. Closed source: observed - # with 2.1.314, recheck on version bumps. + # Claude Code 2.1.327 filters each whitespace-only text block on replay. def replay_view(self, message: Message, dialect: DialectName | None) -> Message: - return drop_blank_text(message) + return drop_blank_text_blocks(message) async def setup(self, runtime: Runtime) -> None: await ensure_node(runtime) @@ -84,6 +85,10 @@ async def prepare_acp( ) -> ACPConfig: system_prompt, prompt = self.resolve_prompt(data) config_dir = self.config_dir(trace) + await runtime.write( + f"{config_dir}/settings.json", + json.dumps({"permissions": CLAUDE_PERMISSIONS}).encode(), + ) await self.install_skills(runtime, f"{config_dir}/skills") versions = {"version": self.config.version, "acp_version": ACP_VERSION} session_meta = { @@ -127,7 +132,7 @@ async def gate_tools( # `CLAUDE_CONFIG_DIR` holds the user settings, which the adapter loads by default. await runtime.write( f"{config.env['CLAUDE_CONFIG_DIR']}/settings.json", - json.dumps({"permissions": {"ask": ["*"]}}).encode(), + json.dumps({"permissions": {**CLAUDE_PERMISSIONS, "ask": ["*"]}}).encode(), ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 0ce3995a00..43852deec4 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,14 +22,19 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" -ACP_VERSION = "2.5.0" +ACP_VERSION = "2.7.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" rm -f {ready} -npm install --prefix {packages} --ignore-scripts --no-audit --no-fund \ +# Resolve the adapter's bundled CLI to the harness pin, including across minor releases. +mkdir -p {packages} +cd {packages} +printf '{"overrides":{"@agentclientprotocol/codex-acp":{"@openai/codex":"%s"}}}\n' \ + "$AGENT_CODEX_VERSION" > package.json +npm install --ignore-scripts --no-audit --no-fund \ --omit=dev \ "@agentclientprotocol/codex-acp@$AGENT_CODEX_ACP_VERSION" \ "@openai/codex@$AGENT_CODEX_VERSION" >/dev/null @@ -48,9 +53,9 @@ class CodexHarnessConfig(HarnessConfig): # Codex accepts no other wire API: - # https://github.com/openai/codex/blob/rust-v0.166.0/codex-rs/model-provider-info/src/lib.rs#L115 + # https://github.com/openai/codex/blob/rust-v0.170.0/codex-rs/model-provider-info/src/lib.rs#L119 dialect: Literal["responses"] = "responses" - version: PinnedVersion = "0.166.0" + version: PinnedVersion = "0.170.0" """Codex release to install, pinned for reproducibility.""" code_mode: bool = False """Enable Code Mode when the model's metadata does not select its tool mode.""" @@ -128,10 +133,15 @@ async def prepare_acp( # Codex reads MCP servers from the config written by build_env(). mcp_urls={}, system_prompt=system_prompt, + # An untitled session starts a separate model turn to generate a UI title. + session_title=f"Verifiers rollout {trace.id}", client_capabilities={ "_meta": { "jetbrains": { - "air": {"version": 1, "capabilities": ["sessionFailure"]} + "air": { + "version": 1, + "capabilities": ["sessionFailure", "sessionIndex"], + } } } }, @@ -216,6 +226,8 @@ async def build_env( } return { **self.config.resolved_env, + # A rollout has no interactive composer that needs suggested follow-ups. + "CODEX_ACP_PROMPT_SUGGESTIONS": "false", "CODEX_CONFIG": json.dumps(config), "CODEX_HOME": home, "DEFAULT_AUTH_REQUEST": json.dumps( diff --git a/verifiers/v1/harnesses/deepseek/harness.py b/verifiers/v1/harnesses/deepseek/harness.py index 69b5254365..a31ee89646 100644 --- a/verifiers/v1/harnesses/deepseek/harness.py +++ b/verifiers/v1/harnesses/deepseek/harness.py @@ -27,9 +27,9 @@ class DeepSeekHarnessConfig(HarnessConfig): # The `deepseek-official` route speaks Anthropic messages: - # https://github.com/deepseek-ai/deepseek-harness/blob/dsh-v0.2.1-rc.5/packages/llm/llm-deepseek/src/messages-api.ts#L14-L17 + # https://github.com/deepseek-ai/deepseek-harness/blob/dsh-v0.2.2-rc.1/packages/llm/llm-deepseek/src/messages-api.ts#L14-L17 dialect: Literal["anthropic_messages"] = "anthropic_messages" - version: PinnedVersion = "0.2.1-rc.5" + version: PinnedVersion = "0.2.2-rc.1" """DeepSeek CLI release, pinned to its shipped sdk-minimal composition.""" @@ -70,7 +70,7 @@ async def prepare_acp( home = f"/tmp/vf-deepseek/{trace.id}" system_prompt, prompt = self.resolve_prompt(data) # Keep the upstream minimal tree; replace only its SDK stdio front door. - # https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.1-rc.5/packages/bundle/sdk-minimal + # https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.2-rc.1/packages/bundle/sdk-minimal patches = [ {"id": "sdk-app-startup", "disabled": True}, {"id": "sdk-jsonrpc-server", "disabled": True}, diff --git a/verifiers/v1/harnesses/hermes_agent/harness.py b/verifiers/v1/harnesses/hermes_agent/harness.py index 61d29739e0..b1689400c3 100644 --- a/verifiers/v1/harnesses/hermes_agent/harness.py +++ b/verifiers/v1/harnesses/hermes_agent/harness.py @@ -24,7 +24,7 @@ curl -fsSL "https://github.com/NousResearch/hermes-agent/archive/refs/tags/$AGENT_HERMES_VERSION.tar.gz" \\ | tar -xz --strip-components=1 -C "$AGENT_HERMES_DIR" # Replay one reasoning key: strict providers reject Hermes' duplicated aliases on custom routes. -# Recheck: https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/message_sanitization.py#L660 +# Recheck: https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/message_sanitization.py#L733 sed -i '/^_REASONING_CARRIER_ROUTES: tuple = (/a\\ (_DEFAULT_CARRIERS, frozenset({{"custom"}}), ()),' "$AGENT_HERMES_DIR/agent/message_sanitization.py" uv sync --project "$AGENT_HERMES_DIR" --python 3.14 --locked --no-dev --extra acp --extra mcp --extra anthropic @@ -33,7 +33,7 @@ class HermesAgentHarnessConfig(HarnessConfig): - version: PinnedVersion = "v0.22.5" + version: PinnedVersion = "v0.23.2" """Hermes Agent Git release tag to install, pinned for reproducibility.""" use_bundled_skill: bool = False """Enable optional bundled skills; Hermes always includes its own operating manual.""" @@ -50,8 +50,8 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]): # Hermes strips stored assistant text and pads missing reasoning with " " on # routes requiring reasoning replay. Recheck on version bumps: - # https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/chat_completion_helpers.py#L1583 - # https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/message_sanitization.py#L867 + # https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/chat_completion_helpers.py#L1563 + # https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/message_sanitization.py#L1007 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return strip_text(message) diff --git a/verifiers/v1/harnesses/kimi_code/harness.py b/verifiers/v1/harnesses/kimi_code/harness.py index 852b999881..9c86eee78c 100644 --- a/verifiers/v1/harnesses/kimi_code/harness.py +++ b/verifiers/v1/harnesses/kimi_code/harness.py @@ -38,7 +38,7 @@ class KimiCodeHarnessConfig(HarnessConfig): - version: PinnedVersion = "2.4.1" + version: PinnedVersion = "2.6.3" """Kimi Code release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" @@ -51,8 +51,8 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]): # Kimi Code drops whitespace-only assistant text when it stores a step, so every # transport replays it without. Recheck on version bumps: - # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.4.1/packages/agent-core-v2/src/agent/loop/loopService.ts#L1819 - # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.4.1/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts#L6 + # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.6.3/packages/agent-core-v2/src/human/llm/policy/settle.ts#L26 + # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.6.3/packages/agent-core-v2/src/human/llm/message.ts#L162-L169 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/mini_swe_agent/program.py b/verifiers/v1/harnesses/mini_swe_agent/program.py index 4fe5171a5a..8a5c3ddacd 100644 --- a/verifiers/v1/harnesses/mini_swe_agent/program.py +++ b/verifiers/v1/harnesses/mini_swe_agent/program.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.10,<3.15" -# dependencies = ["mini-swe-agent=={version}", "litellm[proxy]==1.106.6"] +# dependencies = ["mini-swe-agent=={version}", "litellm[proxy]==1.110.0"] # /// import sys diff --git a/verifiers/v1/harnesses/node.py b/verifiers/v1/harnesses/node.py index 02a32c4340..0d9e9ba7fd 100644 --- a/verifiers/v1/harnesses/node.py +++ b/verifiers/v1/harnesses/node.py @@ -3,13 +3,13 @@ NODE_DIR = "/var/tmp/vf-node" NODE_BIN_DIR = f"{NODE_DIR}/bin" -NODE_VERSION = "22.23.4" +NODE_VERSION = "24.22.3" INSTALL = r""" set -e node=/var/tmp/vf-node # Node bundles Brotli; a newer major alone does not prove CVE-2026-111497 is fixed. -node_ok() { "$1" -e 'const [a,b,c]=process.versions.node.split(".").map(Number); const f={22:[23,4],24:[22,1],26:[12,1]}[a]; process.exit(f && (b>f[0] || b===f[0] && c>=f[1]) ? 0 : 1)'; } +node_ok() { "$1" -e 'const [a,b,c]=process.versions.node.split(".").map(Number); const f={24:[22,1],26:[12,1]}[a]; process.exit(f && (b>f[0] || b===f[0] && c>=f[1]) ? 0 : 1)'; } install_node() { if [ -x "$node/bin/node" ] && [ -x "$node/bin/npm" ] \ @@ -91,7 +91,7 @@ case "$(uname -m)" in aarch64|arm64) node_arch=arm64 ;; *) node_arch=x64 ;; esac install_node "$node_os-$node_arch" fi -node_ok "$node/bin/node" || { echo "ACP adapters require patched Node.js (22.23.4, 24.22.1 or 26.12.1 minimum)" >&2; exit 1; } +node_ok "$node/bin/node" || { echo "ACP adapters require patched Node.js (24.22.1 or 26.12.1 minimum)" >&2; exit 1; } """ diff --git a/verifiers/v1/harnesses/openclaw/harness.py b/verifiers/v1/harnesses/openclaw/harness.py index a36c1b7db1..41c5def328 100644 --- a/verifiers/v1/harnesses/openclaw/harness.py +++ b/verifiers/v1/harnesses/openclaw/harness.py @@ -34,12 +34,15 @@ throw new Error("OpenClaw's transcript sanitizer exceeds the replay patch size limit"); } const source = fs.readFileSync(file, "utf8"); +// The published bundle can rename the local that holds the validated reasoning item. +const item = source.match(/(\w+)\.summary !== void 0 && !Array\.isArray\(\1\.summary\)/)?.[1]; +if (!item) throw new Error("could not locate OpenClaw's reasoning summary validation"); const patches = [ [ "const OPAQUE_REPLAY_TOKEN_RE = /^[A-Za-z0-9+/_-]+={0,2}$/;", "const OPAQUE_REPLAY_TOKEN_RE = /^[A-Za-z0-9+/_-]+={0,2}(?:[.][A-Za-z0-9+/_-]+={0,2})*$/;", ], - ["\t\tsummary: [],", "\t\tsummary: parsed.summary ?? [],"], + ['type: "reasoning",\n\t\tsummary: [],', `type: "reasoning",\n\t\tsummary: ${item}.summary ?? [],`], ]; let patched = source; for (const [before, after] of patches) { @@ -125,7 +128,7 @@ class OpenClawHarnessConfig(HarnessConfig): - version: PinnedVersion = "2026.10.9" + version: PinnedVersion = "2026.11.5" """OpenClaw release to install, pinned for reproducibility.""" dialect: Literal["auto"] = "auto" """OpenClaw takes the transport from its model catalog.""" @@ -219,7 +222,6 @@ async def prepare_acp( "defaults": { "workspace": ".", "skipBootstrap": True, - "heartbeat": {"every": "0m"}, "sandbox": {"mode": "off"}, "model": {"primary": ctx.model}, "utilityModel": "", # Disable background recap model calls. diff --git a/verifiers/v1/harnesses/opencode/harness.py b/verifiers/v1/harnesses/opencode/harness.py index e25752d513..a35c371072 100644 --- a/verifiers/v1/harnesses/opencode/harness.py +++ b/verifiers/v1/harnesses/opencode/harness.py @@ -28,7 +28,7 @@ class OpenCodeHarnessConfig(HarnessConfig): - version: PinnedVersion = "2.2.5" + version: PinnedVersion = "2.3.2" """OpenCode 2 release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" disabled_tools: list[str] | None = None @@ -42,7 +42,7 @@ class OpenCodeHarness(ACPHarness[OpenCodeHarnessConfig]): # OpenCode drops whitespace-only assistant text on replay over Anthropic messages. # Recheck on version bumps: - # https://github.com/anomalyco/opencode/blob/v2.2.5/packages/ai/src/protocols/anthropic-messages.ts#L961 + # https://github.com/anomalyco/opencode/blob/v2.3.2/packages/ai/src/protocols/anthropic-messages.ts#L909 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: if dialect == "anthropic_messages": return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/pi/harness.py b/verifiers/v1/harnesses/pi/harness.py index 26ef2bab81..9bc70c349f 100644 --- a/verifiers/v1/harnesses/pi/harness.py +++ b/verifiers/v1/harnesses/pi/harness.py @@ -51,7 +51,7 @@ class PiHarnessConfig(HarnessConfig): - version: PinnedVersion = "1.4.0" + version: PinnedVersion = "1.5.2" """Pi release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" supports_developer_role: bool | None = None @@ -68,8 +68,8 @@ class PiHarness(ACPHarness[PiHarnessConfig]): # pi-ai drops whitespace-only assistant text on replay over chat completions and # Anthropic messages, not over Responses. Recheck on version bumps: - # https://github.com/earendil-works/pi/blob/v1.4.0/packages/ai/src/api/openai-completions.ts#L1310 - # https://github.com/earendil-works/pi/blob/v1.4.0/packages/ai/src/api/anthropic-messages.ts#L1437 + # https://github.com/earendil-works/pi/blob/v1.5.2/packages/ai/src/api/openai-completions.ts#L1295 + # https://github.com/earendil-works/pi/blob/v1.5.2/packages/ai/src/api/anthropic-messages.ts#L1704 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: if dialect in ("chat_completions", "anthropic_messages"): return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/prime_agent/harness.py b/verifiers/v1/harnesses/prime_agent/harness.py index d530cc8def..11393d30e8 100644 --- a/verifiers/v1/harnesses/prime_agent/harness.py +++ b/verifiers/v1/harnesses/prime_agent/harness.py @@ -21,10 +21,10 @@ GITHUB_RELEASE_URL = ( "https://github.com/PrimeIntellect-ai/prime-agent/releases/download" ) -PRIME_AGENT_COMMIT: Literal["ea9d9602ec0938d9e454e8963107cd85af1565c6"] = ( - "ea9d9602ec0938d9e454e8963107cd85af1565c6" +PRIME_AGENT_COMMIT: Literal["033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe"] = ( + "033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe" ) -PRIME_AGENT_VERSION = "0.10.2" +PRIME_AGENT_VERSION = "0.10.5" PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent" STATE_ROOT = "/tmp/vf-prime-agent-runs" LIFECYCLE_META_NAMESPACE = "ai.primeintellect.prime-agent" @@ -39,10 +39,10 @@ prefix="$AGENT_PRIME_AGENT_DIR/$PRIME_AGENT_COMMIT" if [ ! -x "$prefix/bin/prime-agent" ]; then case "$(uname -s)-$(uname -m)" in - Linux-x86_64) platform=linux-x64; sha=09659961dfde201cc39f1e69b607385d708da3e591f267a4fea04970afb8b005 ;; - Linux-aarch64|Linux-arm64) platform=linux-arm64; sha=cd31de3a52dac3bdee44b723f10341cc5255f135acbcde8894575e324fa852b7 ;; - Darwin-x86_64) platform=darwin-x64; sha=32683ec045940093f6fea66ad234894f0361fd7c466932365c09097edf97499a ;; - Darwin-arm64) platform=darwin-arm64; sha=2f0894811f8ba047eb1874cc1d7392f45c0d6135be6090e7601983de17a6d1d7 ;; + Linux-x86_64) platform=linux-x64; sha=b9f653f4c32e83a2270f0b52edea6a72c7bdbd1f7b5c3e64c55e75157094f3b3 ;; + Linux-aarch64|Linux-arm64) platform=linux-arm64; sha=8ddee5f1ed1a4443829944117509212f1294abbf0cbed22765f8a6efefaa2516 ;; + Darwin-x86_64) platform=darwin-x64; sha=2a19423b21e79ac45050a246a18d507db66801072437e321b1f7063c6bcf9a4b ;; + Darwin-arm64) platform=darwin-arm64; sha=6eb082c66f5259b953c38f21c72055d11942fa61372718a5eab25603247bd49c ;; *) echo "unsupported prime-agent platform: $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac command -v curl >/dev/null || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } || apk add --no-cache curl ca-certificates @@ -76,7 +76,7 @@ class PrimeAgentHarnessConfig(HarnessConfig): - commit: Literal["ea9d9602ec0938d9e454e8963107cd85af1565c6"] = PRIME_AGENT_COMMIT + commit: Literal["033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe"] = PRIME_AGENT_COMMIT """Prime Agent release commit to install.""" autonomous: bool = False @@ -94,7 +94,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]): SUPPORTS_SKILLS = True # Its chat-completions provider drops whitespace-only assistant text on replay. Recheck on bumps: - # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.10.2/crates/pa-ai/src/providers/openai_completions/convert.rs#L106 + # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.10.5/crates/pa-ai/src/providers/openai_completions/convert.rs#L106 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index c83ef17b6b..56a120eeb1 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="695ac48461fbd154859093eaa14589b21f63c22e", min_length=1 + default="8ca926cdd64fd7725b78cf95a4e1667315d3b1f6", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" diff --git a/verifiers/v1/harnesses/stirrup/program.py b/verifiers/v1/harnesses/stirrup/program.py index 32437017f7..662c056560 100644 --- a/verifiers/v1/harnesses/stirrup/program.py +++ b/verifiers/v1/harnesses/stirrup/program.py @@ -2,8 +2,8 @@ # requires-python = ">=3.12,<3.14" # dependencies = [ # "stirrup[mcp] @ https://github.com/ArtificialAnalysis/Stirrup/archive/{version}.tar.gz", -# "mcp==1.28.1", -# "agent-client-protocol==0.12.1", +# "mcp==1.30.0", +# "agent-client-protocol==1.0.0", # ] # /// """Expose Stirrup's agent loop as a rollout-scoped ACP agent over stdio.""" @@ -23,10 +23,10 @@ RequestError, run_agent, stdio_streams, - update_agent_message_text, ) from acp.schema import ( AgentCapabilities, + AgentMessageChunk, CloseSessionResponse, InitializeResponse, McpCapabilities, @@ -34,6 +34,7 @@ PromptResponse, SessionCapabilities, SessionCloseCapabilities, + TextContentBlock, ) from openai import APIError from stirrup import Agent @@ -334,7 +335,7 @@ async def prompt(self, session_id, prompt, **kwargs): return PromptResponse(stop_reason="cancelled") await self.connection.session_update( session_id=session_id, - update=update_agent_message_text(reply), + update=AgentMessageChunk(content=TextContentBlock(text=reply)), ) return PromptResponse(stop_reason="end_turn") diff --git a/verifiers/v1/harnesses/terminus_2/harness.py b/verifiers/v1/harnesses/terminus_2/harness.py index 9f5d092871..9321fd8a15 100644 --- a/verifiers/v1/harnesses/terminus_2/harness.py +++ b/verifiers/v1/harnesses/terminus_2/harness.py @@ -16,7 +16,7 @@ class Terminus2HarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" - version: PinnedVersion = "0.25.0" + version: PinnedVersion = "0.26.0" """Harbor release to install, pinned for reproducibility.""" diff --git a/verifiers/v1/harnesses/utils/replay.py b/verifiers/v1/harnesses/utils/replay.py index 8f1886f4f0..7810b75850 100644 --- a/verifiers/v1/harnesses/utils/replay.py +++ b/verifiers/v1/harnesses/utils/replay.py @@ -2,6 +2,8 @@ `Harness.replay_view` overrides. Each function is idempotent and only normalizes whitespace, so the replayed and the sampled form of a turn map to the same view.""" +from itertools import pairwise + from verifiers.v1.types import AssistantMessage, Message @@ -17,6 +19,23 @@ def drop_blank_text(message: Message) -> Message: return message +def drop_blank_text_blocks(message: Message) -> Message: + """Drop independently blank blocks while preserving whitespace inside other blocks.""" + if not isinstance(message, AssistantMessage) or not message.text_block_offsets: + return drop_blank_text(message) + text = message.content or "" + offsets = (0, *message.text_block_offsets, len(text)) + if any(start > end for start, end in pairwise(offsets)): + return drop_blank_text(message) + blocks = (text[start:end] for start, end in pairwise(offsets)) + return message.model_copy( + update={ + "content": "".join(block for block in blocks if block.strip()) or None, + "text_block_offsets": None, + } + ) + + def strip_text(message: Message) -> Message: """Clients that store assistant text stripped, and pad a missing reasoning field (or Anthropic thinking block) with a single space on replay.""" diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 95b831a1c2..02de183224 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -67,7 +67,7 @@ from verifiers.v1.semantic import ACPInfo, extract_acp_info, routing_session_id from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan -from verifiers.v1.types import FinishReason, Response, Usage +from verifiers.v1.types import FinishReason, Response, ToolMessage, Usage from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.format import format_time from verifiers.v1.utils.logging import clip, get_logger, isolated @@ -881,6 +881,7 @@ async def coalesced( dialect, f"rollout stopped: {refused}", status=400 ) original_request = model_request + harness_history = original_request model_request, request_rewrites, stopped = await session.rewrite_request( model_request, dialect=dialect.name ) @@ -888,6 +889,20 @@ async def coalesced( # A pinned tool result changes the request without a fresh record. if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) + # Match future receipts against the prompt the harness will replay. + # It retains its own user messages. We reapply pinned tool results. + harness_history = original_request.model_copy( + update={ + "messages": [ + after if isinstance(after, ToolMessage) else before + for before, after in zip( + original_request.messages, + model_request.messages, + strict=True, + ) + ] + } + ) except RolloutError as error: return self._fail(session, dialect, error) except Exception as error: # noqa: BLE001 - surface task hook failures @@ -1013,7 +1028,9 @@ async def sample() -> web.Response: session.consume_prepared(turn.tail) session.trace.response_rewrites.extend(response_rewrites) if stopped is None: - stopped = await session.gate_tool_calls(node) + stopped = await session.gate_tool_calls( + node, dialect.name, harness_history + ) if stopped is not None: session.trace.stop(stopped) return _error_response( diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index b06d6f8a55..bba474340d 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -19,7 +19,7 @@ import inspect import json from collections import Counter -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from contextlib import AbstractAsyncContextManager from dataclasses import dataclass, field, replace from functools import cached_property @@ -54,6 +54,7 @@ logger = get_logger(__name__) MCPProxy = Callable[[str], AbstractAsyncContextManager[str]] +ToolResultKey = tuple[int | str, str] def hook_boundary(handler: Callable, *, allow_trace: bool) -> type: @@ -203,20 +204,24 @@ class RolloutSession: """Handler tasks currently serving this session. aiohttp does not cancel a handler when its client disconnects, so a request whose program died at teardown would keep driving the exchange (upstream call, simulator turn) — unregistering cancels these instead.""" - tool_decisions: dict[tuple[str, str], ToolMessage | None] = field( + tool_decisions: dict[ToolResultKey, ToolMessage | None] = field( default_factory=dict ) """The pre-execution verdict per proposed call occurrence: None to run it, else the result the policy put in its place — that call must not execute.""" - pinned_tool_results: dict[tuple[str, str], ToolMessage] = field( - default_factory=dict - ) + pinned_tool_results: dict[ToolResultKey, ToolMessage] = field(default_factory=dict) """The results the model must see, by conversation prefix and call id: rewrites and results it substituted for blocked calls. The harness keeps its own copy of each in its history and re-sends it forever, so every request has these re-applied before the graph is matched — the trace holds the pinned version, and a differing copy would fork it at that node.""" - gated_tools: set[tuple[str, str]] = field(default_factory=set) + gated_tools: set[ToolResultKey] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" + tool_dialects: dict[int, DialectName] = field(default_factory=dict) + """Wire dialect of each issuing node, used when its harness asks the tool gate.""" + tool_issuers: dict[tuple[str, str], set[int]] = field(default_factory=dict) + """Sampled nodes that issued a call ID under a replay-equivalent prefix.""" + tool_catalogs: dict[int, str] = field(default_factory=dict) + """The harness's tool catalog when it requested each issuing turn, before mediation.""" prepared_users: Counter[str] = field(default_factory=Counter) last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and @@ -239,32 +244,73 @@ def ctx(self) -> ModelContext: def stopped(self) -> bool: return self.trace.stop_condition is not None - def tool_result_keys(self, request: Request) -> dict[int, tuple[str, str]]: - """Identify results by their issuing conversation prefix and call id. Canonical - pinned history gives native and rewritten replays the same occurrence keys.""" + def tool_result_keys( + self, + request: Request, + dialect: DialectName | None = None, + known_path: Sequence[int] = (), + ) -> dict[int, ToolResultKey]: + """Use the sampled node and call ID for a model-issued tool. A harness-supplied + call outside the sampled graph uses its conversation prefix instead.""" prefix = hashlib.blake2b(digest_size=16) - calls: dict[str, tuple[str, str]] = {} - keys: dict[int, tuple[str, str]] = {} + calls: dict[str, ToolResultKey] = {} + keys: dict[int, ToolResultKey] = {} + catalog = None for position, message in enumerate(request.messages): if isinstance(message, ToolMessage) and message.tool_call_id in calls: key = calls[message.tool_call_id] + issuer, call_id = key + if isinstance(issuer, str) and ( + candidates := self.tool_issuers.get((issuer, call_id)) + ): + if len(candidates) > 1: + if catalog is None: + catalog = graph.tool_catalog_hash(request.tools) + candidates = { + node + for node in candidates + if self.tool_catalogs[node] == catalog + } + if len(candidates) != 1: + raise HarnessError( + f"Ambiguous issuing turn for tool call {call_id!r}" + ) + key = (next(iter(candidates)), call_id) keys[position] = key message = self.pinned_tool_results.get(key, message) - prefix.update(graph.message_hash(message).encode()) - if isinstance(message, AssistantMessage): - calls.update( - (call.id, (prefix.hexdigest(), call.id)) - for call in message.tool_calls or [] - ) + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) + if isinstance(message, AssistantMessage) and message.tool_calls: + prefix_hash = prefix.hexdigest() + for call in message.tool_calls: + alias = (prefix_hash, call.id) + if ( + position < len(known_path) + and self.trace.nodes[known_path[position]].sampled + ): + issuer = known_path[position] + calls[call.id] = (issuer, call.id) + else: + calls[call.id] = alias return keys - def pin_tool_results(self, request: Request) -> Request: + def pin_tool_results( + self, + request: Request, + dialect: DialectName | None = None, + known_path: Sequence[int] = (), + ) -> Request: """Re-impose the pinned results on the harness's request. A result for a blocked call the harness never gated means the call ran anyway: the transcript would no longer describe the sandbox, so that fails the rollout instead of being papered over.""" + if not self.pinned_tool_results: + return request messages = list(request.messages) changed = False - for position, key in self.tool_result_keys(request).items(): + for position, key in self.tool_result_keys( + request, dialect, known_path + ).items(): pinned = self.pinned_tool_results.get(key) if pinned is None: continue @@ -283,14 +329,15 @@ async def rewrite_request( *, run_stops: bool = True, dialect: DialectName | None = None, + known_path: Sequence[int] = (), ) -> tuple[Request, list[InterceptRecord], str | None]: """Run typed request interceptors and stops over one canonical request. Only the uncommitted tail's user and tool messages may change; a rewritten tool result is pinned for the rest of the rollout.""" - request = self.pin_tool_results(request) - tool_keys = self.tool_result_keys(request) + request = self.pin_tool_results(request, dialect, known_path) if not self.request_interceptors and (not run_stops or not self.request_stops): return request, [], None + tool_keys = self.tool_result_keys(request, dialect, known_path) candidates: set[int] = set() tail_start = graph.message_prefix_len(self.trace, request.messages, dialect) if self.request_interceptors: @@ -399,7 +446,7 @@ async def rewrite_request( f"request interception failed: {type(error).__name__}: {error}" ) from error current = request.model_copy(update={"messages": messages}) - tool_keys = self.tool_result_keys(current) + tool_keys = self.tool_result_keys(current, dialect, known_path) for position in candidates: after = current.messages[position] if ( @@ -422,9 +469,16 @@ async def prepare_users( self, request: Request ) -> tuple[Request, list[InterceptRecord]]: """Intercept caller-owned user turns before the harness stores them.""" - branch = self.trace.messages + known_path = ( + graph.path_node_ids(self.trace, len(self.trace.nodes) - 1) + if self.trace.nodes + else [] + ) + branch = [self.trace.nodes[i].message for i in known_path] rewritten, records, _ = await self.rewrite_request( - Request(messages=[*branch, *request.messages]), run_stops=False + Request(messages=[*branch, *request.messages]), + run_stops=False, + known_path=known_path, ) tail = rewritten.messages[len(branch) :] self.prepared_users.update( @@ -489,27 +543,48 @@ async def rewrite_response( ) from error return response, records, None - async def gate_tool_calls(self, node: int) -> str | None: + async def gate_tool_calls( + self, + node: int, + dialect: DialectName | None = None, + harness_history: Request | None = None, + ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a `@stop` refuses the whole response and a rewrite becomes that call's verdict — the harness's gate denies it and the model sees the rewritten result instead. Returns the name of the stop that fired, if any.""" assistant = self.trace.nodes[node].message - if not (self.request_interceptors or self.request_stops) or not ( - isinstance(assistant, AssistantMessage) and assistant.tool_calls - ): + if not (self.request_interceptors or self.request_stops): + return None + if dialect is not None: + self.tool_dialects[node] = dialect + if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: return None - branch = graph.path(self.trace, node) + known_path = graph.path_node_ids(self.trace, node) + branch = [self.trace.nodes[i].message for i in known_path] + tools = self.trace.nodes[known_path[0]].tools + if harness_history is None: + harness_history = Request(messages=branch[:-1], tools=tools) + self.tool_catalogs[node] = graph.tool_catalog_hash(harness_history.tools) + prefix = hashlib.blake2b(digest_size=16) + for message in [*harness_history.messages, assistant]: + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) + prefix_hash = prefix.hexdigest() for call in assistant.tool_calls: + self.tool_issuers.setdefault((prefix_hash, call.id), set()).add(node) probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) - request = Request(messages=[*branch, probe], tools=self.trace.tools or None) - key = self.tool_result_keys(request)[len(branch)] + request = Request(messages=[*branch, probe], tools=tools or None) + key = (node, call.id) if key in self.tool_decisions: raise HarnessError( f"tool call id {call.id!r} was reused in the same assistant turn" ) - request, records, stopped = await self.rewrite_request(request) + request, records, stopped = await self.rewrite_request( + request, dialect=dialect, known_path=known_path + ) self.trace.request_rewrites.extend(records) if stopped is not None: return stopped @@ -575,11 +650,7 @@ async def decide_tool( ] or matches if len(matches) == 1: leaf, call = matches[0] - branch = graph.path(self.trace, leaf) - request = Request( - messages=[*branch, ToolMessage(tool_call_id=call.id, content="")] - ) - key = self.tool_result_keys(request)[len(branch)] + key = (leaf, call.id) self.gated_tools.add(key) verdict = self.tool_decisions.get(key) elif matches or not name or len(leaves) != 1: @@ -588,7 +659,8 @@ async def decide_tool( f"{len(matches)} proposed calls" ) else: - *branch, assistant = graph.path(self.trace, leaves[0][0]) + known_path = graph.path_node_ids(self.trace, leaves[0][0]) + *branch, assistant = [self.trace.nodes[i].message for i in known_path] assert isinstance(assistant, AssistantMessage) call = ToolCall(id=tool_call_id, name=name, arguments=json.dumps(arguments)) assistant = assistant.model_copy( @@ -597,8 +669,11 @@ async def decide_tool( probe = ToolMessage(tool_call_id=call.id, content="", name=name) request, records, stopped = await self.rewrite_request( Request( - messages=[*branch, assistant, probe], tools=self.trace.tools or None - ) + messages=[*branch, assistant, probe], + tools=self.trace.nodes[known_path[0]].tools or None, + ), + dialect=self.tool_dialects.get(leaves[0][0]), + known_path=known_path[:-1], ) self.trace.request_rewrites.extend(records) if stopped is not None: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 6bcbe22179..e695a3aaca 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -742,12 +742,7 @@ def branches(self) -> list[Branch]: } branches: list[Branch] = [] for i, leaf in enumerate(graph.leaves(self)): - path: list[int] = [] - nid: int | None = leaf - while nid is not None: - path.append(nid) - nid = self.nodes[nid].parent - path.reverse() + path = graph.path_node_ids(self, leaf) is_compaction_attempt = any( link.type == "compaction_attempt" for link in self.nodes[leaf].semantic_parents diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index 9961ef4ea1..d67782c39a 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -136,6 +136,10 @@ class ToolCall(BaseModel): class AssistantMessage(BaseModel): role: Literal["assistant"] = "assistant" content: str | None = None + text_block_offsets: list[int] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + """Character offsets between native text blocks concatenated into content.""" reasoning_content: str | None = None tool_calls: list[ToolCall] | None = None provider_state: list[dict[str, Any]] | None = None