From abe8271aef393c9000ee6d658a82d511e1e64156 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:27:29 -0800 Subject: [PATCH 01/13] Update bundled harness releases and ACP adapters --- tests/integration/grid.py | 2 +- verifiers/v1/harnesses/claude_code/harness.py | 4 ++-- verifiers/v1/harnesses/codex/harness.py | 10 +++++++--- verifiers/v1/harnesses/deepseek/harness.py | 6 +++--- verifiers/v1/harnesses/hermes_agent/harness.py | 8 ++++---- verifiers/v1/harnesses/kimi_code/harness.py | 6 +++--- .../v1/harnesses/mini_swe_agent/program.py | 2 +- verifiers/v1/harnesses/node.py | 6 +++--- verifiers/v1/harnesses/openclaw/harness.py | 7 +++++-- verifiers/v1/harnesses/opencode/harness.py | 4 ++-- verifiers/v1/harnesses/pi/harness.py | 6 +++--- verifiers/v1/harnesses/prime_agent/harness.py | 18 +++++++++--------- verifiers/v1/harnesses/rlm/harness.py | 2 +- verifiers/v1/harnesses/stirrup/program.py | 9 +++++---- verifiers/v1/harnesses/terminus_2/harness.py | 2 +- 15 files changed, 50 insertions(+), 42 deletions(-) diff --git a/tests/integration/grid.py b/tests/integration/grid.py index b93a443bfe..03154d421f 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -83,7 +83,7 @@ "mini-swe-agent": "https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/models/litellm_model.py#L79-L90", # Drops historical runtime-context messages and can relocate the current # prompt's context around user turns. - "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.10.9/src/agents/internal-runtime-context.ts#L403-L513", + "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.11.5/src/agents/internal-runtime-context.ts#L445-L563", } diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 35451a0449..0cd86a64a4 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -16,7 +16,7 @@ CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp-{version}-{acp_version}" PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages" -ACP_VERSION = "0.99.1" +ACP_VERSION = "0.108.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" CLAUDE_CONFIG_ROOT = ".vf-claude" @@ -34,7 +34,7 @@ class ClaudeCodeHarnessConfig(HarnessConfig): dialect: Literal["anthropic_messages"] = "anthropic_messages" - version: PinnedVersion = "2.1.314" + version: PinnedVersion = "2.1.327" """Claude Code release to install, pinned for reproducibility.""" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 0ce3995a00..8509c3c9cb 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,13 +22,17 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" -ACP_VERSION = "2.5.0" +ACP_VERSION = "2.7.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" rm -f {ready} +# Resolve the adapter's bundled CLI to the harness pin, including across minor releases. +mkdir -p {packages} +printf '{"overrides":{"@agentclientprotocol/codex-acp":{"@openai/codex":"%s"}}}\n' \ + "$AGENT_CODEX_VERSION" > {packages}/package.json npm install --prefix {packages} --ignore-scripts --no-audit --no-fund \ --omit=dev \ "@agentclientprotocol/codex-acp@$AGENT_CODEX_ACP_VERSION" \ @@ -48,9 +52,9 @@ class CodexHarnessConfig(HarnessConfig): # Codex accepts no other wire API: - # https://github.com/openai/codex/blob/rust-v0.166.0/codex-rs/model-provider-info/src/lib.rs#L115 + # https://github.com/openai/codex/blob/rust-v0.170.0/codex-rs/model-provider-info/src/lib.rs#L119 dialect: Literal["responses"] = "responses" - version: PinnedVersion = "0.166.0" + version: PinnedVersion = "0.170.0" """Codex release to install, pinned for reproducibility.""" code_mode: bool = False """Enable Code Mode when the model's metadata does not select its tool mode.""" diff --git a/verifiers/v1/harnesses/deepseek/harness.py b/verifiers/v1/harnesses/deepseek/harness.py index 69b5254365..a31ee89646 100644 --- a/verifiers/v1/harnesses/deepseek/harness.py +++ b/verifiers/v1/harnesses/deepseek/harness.py @@ -27,9 +27,9 @@ class DeepSeekHarnessConfig(HarnessConfig): # The `deepseek-official` route speaks Anthropic messages: - # https://github.com/deepseek-ai/deepseek-harness/blob/dsh-v0.2.1-rc.5/packages/llm/llm-deepseek/src/messages-api.ts#L14-L17 + # https://github.com/deepseek-ai/deepseek-harness/blob/dsh-v0.2.2-rc.1/packages/llm/llm-deepseek/src/messages-api.ts#L14-L17 dialect: Literal["anthropic_messages"] = "anthropic_messages" - version: PinnedVersion = "0.2.1-rc.5" + version: PinnedVersion = "0.2.2-rc.1" """DeepSeek CLI release, pinned to its shipped sdk-minimal composition.""" @@ -70,7 +70,7 @@ async def prepare_acp( home = f"/tmp/vf-deepseek/{trace.id}" system_prompt, prompt = self.resolve_prompt(data) # Keep the upstream minimal tree; replace only its SDK stdio front door. - # https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.1-rc.5/packages/bundle/sdk-minimal + # https://github.com/deepseek-ai/deepseek-harness/tree/dsh-v0.2.2-rc.1/packages/bundle/sdk-minimal patches = [ {"id": "sdk-app-startup", "disabled": True}, {"id": "sdk-jsonrpc-server", "disabled": True}, diff --git a/verifiers/v1/harnesses/hermes_agent/harness.py b/verifiers/v1/harnesses/hermes_agent/harness.py index 61d29739e0..b1689400c3 100644 --- a/verifiers/v1/harnesses/hermes_agent/harness.py +++ b/verifiers/v1/harnesses/hermes_agent/harness.py @@ -24,7 +24,7 @@ curl -fsSL "https://github.com/NousResearch/hermes-agent/archive/refs/tags/$AGENT_HERMES_VERSION.tar.gz" \\ | tar -xz --strip-components=1 -C "$AGENT_HERMES_DIR" # Replay one reasoning key: strict providers reject Hermes' duplicated aliases on custom routes. -# Recheck: https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/message_sanitization.py#L660 +# Recheck: https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/message_sanitization.py#L733 sed -i '/^_REASONING_CARRIER_ROUTES: tuple = (/a\\ (_DEFAULT_CARRIERS, frozenset({{"custom"}}), ()),' "$AGENT_HERMES_DIR/agent/message_sanitization.py" uv sync --project "$AGENT_HERMES_DIR" --python 3.14 --locked --no-dev --extra acp --extra mcp --extra anthropic @@ -33,7 +33,7 @@ class HermesAgentHarnessConfig(HarnessConfig): - version: PinnedVersion = "v0.22.5" + version: PinnedVersion = "v0.23.2" """Hermes Agent Git release tag to install, pinned for reproducibility.""" use_bundled_skill: bool = False """Enable optional bundled skills; Hermes always includes its own operating manual.""" @@ -50,8 +50,8 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]): # Hermes strips stored assistant text and pads missing reasoning with " " on # routes requiring reasoning replay. Recheck on version bumps: - # https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/chat_completion_helpers.py#L1583 - # https://github.com/NousResearch/hermes-agent/blob/v0.22.5/agent/message_sanitization.py#L867 + # https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/chat_completion_helpers.py#L1563 + # https://github.com/NousResearch/hermes-agent/blob/v0.23.2/agent/message_sanitization.py#L1007 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return strip_text(message) diff --git a/verifiers/v1/harnesses/kimi_code/harness.py b/verifiers/v1/harnesses/kimi_code/harness.py index 852b999881..9c86eee78c 100644 --- a/verifiers/v1/harnesses/kimi_code/harness.py +++ b/verifiers/v1/harnesses/kimi_code/harness.py @@ -38,7 +38,7 @@ class KimiCodeHarnessConfig(HarnessConfig): - version: PinnedVersion = "2.4.1" + version: PinnedVersion = "2.6.3" """Kimi Code release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" @@ -51,8 +51,8 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]): # Kimi Code drops whitespace-only assistant text when it stores a step, so every # transport replays it without. Recheck on version bumps: - # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.4.1/packages/agent-core-v2/src/agent/loop/loopService.ts#L1819 - # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.4.1/packages/agent-core-v2/src/agent/contextMemory/vacuousContent.ts#L6 + # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.6.3/packages/agent-core-v2/src/human/llm/policy/settle.ts#L26 + # https://github.com/MoonshotAI/kimi-code/blob/@moonshot-ai/kimi-code@2.6.3/packages/agent-core-v2/src/human/llm/message.ts#L162-L169 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/mini_swe_agent/program.py b/verifiers/v1/harnesses/mini_swe_agent/program.py index 4fe5171a5a..8a5c3ddacd 100644 --- a/verifiers/v1/harnesses/mini_swe_agent/program.py +++ b/verifiers/v1/harnesses/mini_swe_agent/program.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.10,<3.15" -# dependencies = ["mini-swe-agent=={version}", "litellm[proxy]==1.106.6"] +# dependencies = ["mini-swe-agent=={version}", "litellm[proxy]==1.110.0"] # /// import sys diff --git a/verifiers/v1/harnesses/node.py b/verifiers/v1/harnesses/node.py index 02a32c4340..0d9e9ba7fd 100644 --- a/verifiers/v1/harnesses/node.py +++ b/verifiers/v1/harnesses/node.py @@ -3,13 +3,13 @@ NODE_DIR = "/var/tmp/vf-node" NODE_BIN_DIR = f"{NODE_DIR}/bin" -NODE_VERSION = "22.23.4" +NODE_VERSION = "24.22.3" INSTALL = r""" set -e node=/var/tmp/vf-node # Node bundles Brotli; a newer major alone does not prove CVE-2026-111497 is fixed. -node_ok() { "$1" -e 'const [a,b,c]=process.versions.node.split(".").map(Number); const f={22:[23,4],24:[22,1],26:[12,1]}[a]; process.exit(f && (b>f[0] || b===f[0] && c>=f[1]) ? 0 : 1)'; } +node_ok() { "$1" -e 'const [a,b,c]=process.versions.node.split(".").map(Number); const f={24:[22,1],26:[12,1]}[a]; process.exit(f && (b>f[0] || b===f[0] && c>=f[1]) ? 0 : 1)'; } install_node() { if [ -x "$node/bin/node" ] && [ -x "$node/bin/npm" ] \ @@ -91,7 +91,7 @@ case "$(uname -m)" in aarch64|arm64) node_arch=arm64 ;; *) node_arch=x64 ;; esac install_node "$node_os-$node_arch" fi -node_ok "$node/bin/node" || { echo "ACP adapters require patched Node.js (22.23.4, 24.22.1 or 26.12.1 minimum)" >&2; exit 1; } +node_ok "$node/bin/node" || { echo "ACP adapters require patched Node.js (24.22.1 or 26.12.1 minimum)" >&2; exit 1; } """ diff --git a/verifiers/v1/harnesses/openclaw/harness.py b/verifiers/v1/harnesses/openclaw/harness.py index a36c1b7db1..b85bbbb270 100644 --- a/verifiers/v1/harnesses/openclaw/harness.py +++ b/verifiers/v1/harnesses/openclaw/harness.py @@ -34,12 +34,15 @@ throw new Error("OpenClaw's transcript sanitizer exceeds the replay patch size limit"); } const source = fs.readFileSync(file, "utf8"); +// The published bundle can rename the local that holds the validated reasoning item. +const item = source.match(/(\w+)\.summary !== void 0 && !Array\.isArray\(\1\.summary\)/)?.[1]; +if (!item) throw new Error("could not locate OpenClaw's reasoning summary validation"); const patches = [ [ "const OPAQUE_REPLAY_TOKEN_RE = /^[A-Za-z0-9+/_-]+={0,2}$/;", "const OPAQUE_REPLAY_TOKEN_RE = /^[A-Za-z0-9+/_-]+={0,2}(?:[.][A-Za-z0-9+/_-]+={0,2})*$/;", ], - ["\t\tsummary: [],", "\t\tsummary: parsed.summary ?? [],"], + ['type: "reasoning",\n\t\tsummary: [],', `type: "reasoning",\n\t\tsummary: ${item}.summary ?? [],`], ]; let patched = source; for (const [before, after] of patches) { @@ -125,7 +128,7 @@ class OpenClawHarnessConfig(HarnessConfig): - version: PinnedVersion = "2026.10.9" + version: PinnedVersion = "2026.11.5" """OpenClaw release to install, pinned for reproducibility.""" dialect: Literal["auto"] = "auto" """OpenClaw takes the transport from its model catalog.""" diff --git a/verifiers/v1/harnesses/opencode/harness.py b/verifiers/v1/harnesses/opencode/harness.py index e25752d513..a35c371072 100644 --- a/verifiers/v1/harnesses/opencode/harness.py +++ b/verifiers/v1/harnesses/opencode/harness.py @@ -28,7 +28,7 @@ class OpenCodeHarnessConfig(HarnessConfig): - version: PinnedVersion = "2.2.5" + version: PinnedVersion = "2.3.2" """OpenCode 2 release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" disabled_tools: list[str] | None = None @@ -42,7 +42,7 @@ class OpenCodeHarness(ACPHarness[OpenCodeHarnessConfig]): # OpenCode drops whitespace-only assistant text on replay over Anthropic messages. # Recheck on version bumps: - # https://github.com/anomalyco/opencode/blob/v2.2.5/packages/ai/src/protocols/anthropic-messages.ts#L961 + # https://github.com/anomalyco/opencode/blob/v2.3.2/packages/ai/src/protocols/anthropic-messages.ts#L909 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: if dialect == "anthropic_messages": return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/pi/harness.py b/verifiers/v1/harnesses/pi/harness.py index 26ef2bab81..9bc70c349f 100644 --- a/verifiers/v1/harnesses/pi/harness.py +++ b/verifiers/v1/harnesses/pi/harness.py @@ -51,7 +51,7 @@ class PiHarnessConfig(HarnessConfig): - version: PinnedVersion = "1.4.0" + version: PinnedVersion = "1.5.2" """Pi release to install, pinned for reproducibility.""" dialect: DialectName = "chat_completions" supports_developer_role: bool | None = None @@ -68,8 +68,8 @@ class PiHarness(ACPHarness[PiHarnessConfig]): # pi-ai drops whitespace-only assistant text on replay over chat completions and # Anthropic messages, not over Responses. Recheck on version bumps: - # https://github.com/earendil-works/pi/blob/v1.4.0/packages/ai/src/api/openai-completions.ts#L1310 - # https://github.com/earendil-works/pi/blob/v1.4.0/packages/ai/src/api/anthropic-messages.ts#L1437 + # https://github.com/earendil-works/pi/blob/v1.5.2/packages/ai/src/api/openai-completions.ts#L1295 + # https://github.com/earendil-works/pi/blob/v1.5.2/packages/ai/src/api/anthropic-messages.ts#L1704 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: if dialect in ("chat_completions", "anthropic_messages"): return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/prime_agent/harness.py b/verifiers/v1/harnesses/prime_agent/harness.py index cbdc958ac4..2e4a2e98ad 100644 --- a/verifiers/v1/harnesses/prime_agent/harness.py +++ b/verifiers/v1/harnesses/prime_agent/harness.py @@ -21,10 +21,10 @@ GITHUB_RELEASE_URL = ( "https://github.com/PrimeIntellect-ai/prime-agent/releases/download" ) -PRIME_AGENT_COMMIT: Literal["ea9d9602ec0938d9e454e8963107cd85af1565c6"] = ( - "ea9d9602ec0938d9e454e8963107cd85af1565c6" +PRIME_AGENT_COMMIT: Literal["033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe"] = ( + "033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe" ) -PRIME_AGENT_VERSION = "0.10.2" +PRIME_AGENT_VERSION = "0.10.5" PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent" STATE_ROOT = "/tmp/vf-prime-agent-runs" LIFECYCLE_META_NAMESPACE = "ai.primeintellect.prime-agent" @@ -39,10 +39,10 @@ prefix="$AGENT_PRIME_AGENT_DIR/$PRIME_AGENT_COMMIT" if [ ! -x "$prefix/bin/prime-agent" ]; then case "$(uname -s)-$(uname -m)" in - Linux-x86_64) platform=linux-x64; sha=09659961dfde201cc39f1e69b607385d708da3e591f267a4fea04970afb8b005 ;; - Linux-aarch64|Linux-arm64) platform=linux-arm64; sha=cd31de3a52dac3bdee44b723f10341cc5255f135acbcde8894575e324fa852b7 ;; - Darwin-x86_64) platform=darwin-x64; sha=32683ec045940093f6fea66ad234894f0361fd7c466932365c09097edf97499a ;; - Darwin-arm64) platform=darwin-arm64; sha=2f0894811f8ba047eb1874cc1d7392f45c0d6135be6090e7601983de17a6d1d7 ;; + Linux-x86_64) platform=linux-x64; sha=b9f653f4c32e83a2270f0b52edea6a72c7bdbd1f7b5c3e64c55e75157094f3b3 ;; + Linux-aarch64|Linux-arm64) platform=linux-arm64; sha=8ddee5f1ed1a4443829944117509212f1294abbf0cbed22765f8a6efefaa2516 ;; + Darwin-x86_64) platform=darwin-x64; sha=2a19423b21e79ac45050a246a18d507db66801072437e321b1f7063c6bcf9a4b ;; + Darwin-arm64) platform=darwin-arm64; sha=6eb082c66f5259b953c38f21c72055d11942fa61372718a5eab25603247bd49c ;; *) echo "unsupported prime-agent platform: $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac command -v curl >/dev/null || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } || apk add --no-cache curl ca-certificates @@ -76,7 +76,7 @@ class PrimeAgentHarnessConfig(HarnessConfig): - commit: Literal["ea9d9602ec0938d9e454e8963107cd85af1565c6"] = PRIME_AGENT_COMMIT + commit: Literal["033eea1705c5ae953fe7cd7a5e7e7c45317e0dfe"] = PRIME_AGENT_COMMIT """Prime Agent release commit to install.""" autonomous: bool = False @@ -90,7 +90,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]): SUPPORTS_SKILLS = True # Its chat-completions provider drops whitespace-only assistant text on replay. Recheck on bumps: - # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.10.2/crates/pa-ai/src/providers/openai_completions/convert.rs#L106 + # https://github.com/PrimeIntellect-ai/prime-agent/blob/v0.10.5/crates/pa-ai/src/providers/openai_completions/convert.rs#L106 def replay_view(self, message: Message, dialect: DialectName | None) -> Message: return drop_blank_text(message) diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index c83ef17b6b..56a120eeb1 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="695ac48461fbd154859093eaa14589b21f63c22e", min_length=1 + default="8ca926cdd64fd7725b78cf95a4e1667315d3b1f6", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" diff --git a/verifiers/v1/harnesses/stirrup/program.py b/verifiers/v1/harnesses/stirrup/program.py index 32437017f7..662c056560 100644 --- a/verifiers/v1/harnesses/stirrup/program.py +++ b/verifiers/v1/harnesses/stirrup/program.py @@ -2,8 +2,8 @@ # requires-python = ">=3.12,<3.14" # dependencies = [ # "stirrup[mcp] @ https://github.com/ArtificialAnalysis/Stirrup/archive/{version}.tar.gz", -# "mcp==1.28.1", -# "agent-client-protocol==0.12.1", +# "mcp==1.30.0", +# "agent-client-protocol==1.0.0", # ] # /// """Expose Stirrup's agent loop as a rollout-scoped ACP agent over stdio.""" @@ -23,10 +23,10 @@ RequestError, run_agent, stdio_streams, - update_agent_message_text, ) from acp.schema import ( AgentCapabilities, + AgentMessageChunk, CloseSessionResponse, InitializeResponse, McpCapabilities, @@ -34,6 +34,7 @@ PromptResponse, SessionCapabilities, SessionCloseCapabilities, + TextContentBlock, ) from openai import APIError from stirrup import Agent @@ -334,7 +335,7 @@ async def prompt(self, session_id, prompt, **kwargs): return PromptResponse(stop_reason="cancelled") await self.connection.session_update( session_id=session_id, - update=update_agent_message_text(reply), + update=AgentMessageChunk(content=TextContentBlock(text=reply)), ) return PromptResponse(stop_reason="end_turn") diff --git a/verifiers/v1/harnesses/terminus_2/harness.py b/verifiers/v1/harnesses/terminus_2/harness.py index 9f5d092871..9321fd8a15 100644 --- a/verifiers/v1/harnesses/terminus_2/harness.py +++ b/verifiers/v1/harnesses/terminus_2/harness.py @@ -16,7 +16,7 @@ class Terminus2HarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" - version: PinnedVersion = "0.25.0" + version: PinnedVersion = "0.26.0" """Harbor release to install, pinned for reproducibility.""" From 80d6d82588dad37a457bafd62b4955a8ca0e8d64 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:44:52 -0800 Subject: [PATCH 02/13] Fix OpenClaw startup and use manual ACP approvals in Claude Code --- verifiers/v1/harnesses/claude_code/harness.py | 8 +++++++- verifiers/v1/harnesses/openclaw/harness.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 0cd86a64a4..6b0cf39cee 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -20,6 +20,8 @@ CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" CLAUDE_CONFIG_ROOT = ".vf-claude" +# Manual mode routes tool approvals to ACP. Auto needs a separate classifier. +CLAUDE_PERMISSIONS = {"defaultMode": "default"} ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -84,6 +86,10 @@ async def prepare_acp( ) -> ACPConfig: system_prompt, prompt = self.resolve_prompt(data) config_dir = self.config_dir(trace) + await runtime.write( + f"{config_dir}/settings.json", + json.dumps({"permissions": CLAUDE_PERMISSIONS}).encode(), + ) await self.install_skills(runtime, f"{config_dir}/skills") versions = {"version": self.config.version, "acp_version": ACP_VERSION} session_meta = { @@ -127,7 +133,7 @@ async def gate_tools( # `CLAUDE_CONFIG_DIR` holds the user settings, which the adapter loads by default. await runtime.write( f"{config.env['CLAUDE_CONFIG_DIR']}/settings.json", - json.dumps({"permissions": {"ask": ["*"]}}).encode(), + json.dumps({"permissions": {**CLAUDE_PERMISSIONS, "ask": ["*"]}}).encode(), ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: diff --git a/verifiers/v1/harnesses/openclaw/harness.py b/verifiers/v1/harnesses/openclaw/harness.py index b85bbbb270..41c5def328 100644 --- a/verifiers/v1/harnesses/openclaw/harness.py +++ b/verifiers/v1/harnesses/openclaw/harness.py @@ -222,7 +222,6 @@ async def prepare_acp( "defaults": { "workspace": ".", "skipBootstrap": True, - "heartbeat": {"every": "0m"}, "sandbox": {"mode": "off"}, "model": {"primary": ctx.model}, "utilityModel": "", # Disable background recap model calls. From b67b372c4bcc29fff80db114b5adba8f03d2bf6b Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:56:34 -0800 Subject: [PATCH 03/13] Diagnose rejected Responses item shapes in the resume integration cell --- tests/integration/test_resume_tool.py | 59 ++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_resume_tool.py b/tests/integration/test_resume_tool.py index 58727e1770..b20e5a4944 100644 --- a/tests/integration/test_resume_tool.py +++ b/tests/integration/test_resume_tool.py @@ -1,15 +1,63 @@ """One task, two segments: the harness preserves context and MCP access across a resumed exchange (echo-acp-resume), across the harness x runtime grid.""" +import re + import pytest from grid import assert_one_branch, grid +from pydantic_core import to_json + +from verifiers.v1.clients.eval import EvalClient @pytest.mark.parametrize( "harness,harness_runtime", grid(needs=("resume", "mcp")), indirect=True ) -async def test_resume_tool(run, harness, harness_runtime): +async def test_resume_tool(run, harness, harness_runtime, monkeypatch): """Each harness preserves context and MCP access across two segments.""" + if harness.id == "codex": + send = EvalClient._request + + async def inspect_rejection(self, url, body, headers, **kwargs): + try: + return await send(self, url, body, headers, **kwargs) + except Exception as error: + match = re.search(r"InputItemWire at line 1 column (\d+)", str(error)) + if match and isinstance(body.get("input"), list): + wire = to_json(body, inf_nan_mode="null") + prefix = b'"input":[' + offset = wire.index(prefix) + len(prefix) + for index, item in enumerate(body["input"]): + end = offset + len(to_json(item, inf_nan_mode="null")) + fields = { + key: type(value).__name__ for key, value in item.items() + } + tags = { + key: item[key] + for key in ("type", "role", "phase", "status") + if key in item + } + children = { + key: [p.get("type") for p in value if isinstance(p, dict)][ + :10 + ] + for key, value in item.items() + if isinstance(value, list) + } + print( + "RESPONSES INPUT", + index, + "bytes", + offset + 1, + end, + tags, + fields, + children, + ) + offset = end + 1 + raise + + monkeypatch.setattr(EvalClient, "_request", inspect_rejection) (episode,) = await run( "echo-acp-resume", harness=harness, @@ -24,7 +72,14 @@ async def test_resume_tool(run, harness, harness_runtime): assert_one_branch(trace, harness, forks=("openclaw",)) assert trace.ok, [error.message for error in trace.errors] assert trace.stop_condition == "user_closed" - assert trace.rewards["resumed"].score == 1.0 + assert trace.rewards["resumed"].score == 1.0, [ + ( + segment["roles"], + [value[:240] for value in segment["tool_outputs"]], + segment["last_reply"][:240], + ) + for segment in trace.info["acp_segments"] + ] segments = trace.info["acp_segments"] assert len(segments) == 2 assert segments[0]["terminated"] is False From ae99bb96304e9664b843af8b45d22cc47b6440d3 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:02:45 -0800 Subject: [PATCH 04/13] Isolate the rejected catalog on the CI provider --- tests/integration/test_resume_tool.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/test_resume_tool.py b/tests/integration/test_resume_tool.py index b20e5a4944..282bef3545 100644 --- a/tests/integration/test_resume_tool.py +++ b/tests/integration/test_resume_tool.py @@ -8,6 +8,7 @@ from pydantic_core import to_json from verifiers.v1.clients.eval import EvalClient +from verifiers.v1.errors import ProviderError @pytest.mark.parametrize( @@ -55,6 +56,28 @@ async def inspect_rejection(self, url, body, headers, **kwargs): children, ) offset = end + 1 + items = body["input"] + print( + "ACP title request", + "title for a conversation" in str(items[-1]), + ) + for label, sample in [ + ("without catalog", items[1:]), + ( + "catalog alone", + [items[0], {"role": "user", "content": "Reply OK."}], + ), + ("control", [{"role": "user", "content": "Reply OK."}]), + ]: + try: + probe = await send( + self, url, {**body, "input": sample}, headers, **kwargs + ) + except ProviderError as probe_error: + print("PROVIDER A/B", label, str(probe_error)[:400]) + else: + print("PROVIDER A/B", label, probe.status_code) + await probe.aclose() raise monkeypatch.setattr(EvalClient, "_request", inspect_rejection) From 751562f8e100c1277699a5bdc1d2abc16e6abcbe Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:06:26 -0800 Subject: [PATCH 05/13] Keep Codex UI title and follow-up generation out of rollouts --- tests/integration/fixtures/network_policy.py | 6 +- tests/integration/test_resume_tool.py | 82 +------------------- verifiers/v1/acp/__init__.py | 2 + verifiers/v1/acp/runner.py | 4 + verifiers/v1/harnesses/codex/harness.py | 9 ++- 5 files changed, 19 insertions(+), 84 deletions(-) diff --git a/tests/integration/fixtures/network_policy.py b/tests/integration/fixtures/network_policy.py index 8e6489cce8..47e86a7b7f 100644 --- a/tests/integration/fixtures/network_policy.py +++ b/tests/integration/fixtures/network_policy.py @@ -68,9 +68,9 @@ def load(self) -> list[NetworkPolicyTask]: "allow": {"network": vf.NetworkPolicyConfig(allow=[ALLOWED])}, } prompt = ( - f"Your only task: run the command `python3 {PROBE_PATH}` once in your " - "shell, then reply with exactly what it printed. Any network policy is " - "already enforced, so do not inspect, configure, or change the network or " + f"Use your shell tool now to run `python3 {PROBE_PATH}` exactly once. " + "Wait for the command to finish, then reply with exactly what it printed. " + "The network policy is already enforced, so do not inspect or change the network or " "firewall, and do not run any other command." ) return [ diff --git a/tests/integration/test_resume_tool.py b/tests/integration/test_resume_tool.py index 282bef3545..58727e1770 100644 --- a/tests/integration/test_resume_tool.py +++ b/tests/integration/test_resume_tool.py @@ -1,86 +1,15 @@ """One task, two segments: the harness preserves context and MCP access across a resumed exchange (echo-acp-resume), across the harness x runtime grid.""" -import re - import pytest from grid import assert_one_branch, grid -from pydantic_core import to_json - -from verifiers.v1.clients.eval import EvalClient -from verifiers.v1.errors import ProviderError @pytest.mark.parametrize( "harness,harness_runtime", grid(needs=("resume", "mcp")), indirect=True ) -async def test_resume_tool(run, harness, harness_runtime, monkeypatch): +async def test_resume_tool(run, harness, harness_runtime): """Each harness preserves context and MCP access across two segments.""" - if harness.id == "codex": - send = EvalClient._request - - async def inspect_rejection(self, url, body, headers, **kwargs): - try: - return await send(self, url, body, headers, **kwargs) - except Exception as error: - match = re.search(r"InputItemWire at line 1 column (\d+)", str(error)) - if match and isinstance(body.get("input"), list): - wire = to_json(body, inf_nan_mode="null") - prefix = b'"input":[' - offset = wire.index(prefix) + len(prefix) - for index, item in enumerate(body["input"]): - end = offset + len(to_json(item, inf_nan_mode="null")) - fields = { - key: type(value).__name__ for key, value in item.items() - } - tags = { - key: item[key] - for key in ("type", "role", "phase", "status") - if key in item - } - children = { - key: [p.get("type") for p in value if isinstance(p, dict)][ - :10 - ] - for key, value in item.items() - if isinstance(value, list) - } - print( - "RESPONSES INPUT", - index, - "bytes", - offset + 1, - end, - tags, - fields, - children, - ) - offset = end + 1 - items = body["input"] - print( - "ACP title request", - "title for a conversation" in str(items[-1]), - ) - for label, sample in [ - ("without catalog", items[1:]), - ( - "catalog alone", - [items[0], {"role": "user", "content": "Reply OK."}], - ), - ("control", [{"role": "user", "content": "Reply OK."}]), - ]: - try: - probe = await send( - self, url, {**body, "input": sample}, headers, **kwargs - ) - except ProviderError as probe_error: - print("PROVIDER A/B", label, str(probe_error)[:400]) - else: - print("PROVIDER A/B", label, probe.status_code) - await probe.aclose() - raise - - monkeypatch.setattr(EvalClient, "_request", inspect_rejection) (episode,) = await run( "echo-acp-resume", harness=harness, @@ -95,14 +24,7 @@ async def inspect_rejection(self, url, body, headers, **kwargs): assert_one_branch(trace, harness, forks=("openclaw",)) assert trace.ok, [error.message for error in trace.errors] assert trace.stop_condition == "user_closed" - assert trace.rewards["resumed"].score == 1.0, [ - ( - segment["roles"], - [value[:240] for value in segment["tool_outputs"]], - segment["last_reply"][:240], - ) - for segment in trace.info["acp_segments"] - ] + assert trace.rewards["resumed"].score == 1.0 segments = trace.info["acp_segments"] assert len(segments) == 2 assert segments[0]["terminated"] is False diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 8f35eb6551..eed94416ff 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -65,6 +65,7 @@ class ACPConfig: system_prompt: str | None = None session_meta: JsonObject | None = None client_capabilities: JsonObject | None = None + session_title: str | None = None class ACPHarness(Harness[ConfigT]): @@ -335,6 +336,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult: "system_prompt": self.config.system_prompt or "", "session_meta": self.config.session_meta or {}, "client_capabilities": self.config.client_capabilities or {}, + "session_title": self.config.session_title, "tool_interception": { "url": self.tool_interception_url, "secret": self.secret, diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index 021729f19c..997b39bbce 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -272,6 +272,10 @@ async def start(self, config: dict) -> None: mcp_servers=mcp_servers(config), **config["session_meta"], ) + if title := config.get("session_title"): + await self.connection.ext_method( + "session/rename", {"sessionId": session.session_id, "title": title} + ) except BaseException: with suppress(BaseException): await self.stack.aclose() diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 8509c3c9cb..79bcec4026 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -132,10 +132,15 @@ async def prepare_acp( # Codex reads MCP servers from the config written by build_env(). mcp_urls={}, system_prompt=system_prompt, + # An untitled session starts a separate model turn to generate a UI title. + session_title=f"Verifiers rollout {trace.id}", client_capabilities={ "_meta": { "jetbrains": { - "air": {"version": 1, "capabilities": ["sessionFailure"]} + "air": { + "version": 1, + "capabilities": ["sessionFailure", "sessionIndex"], + } } } }, @@ -220,6 +225,8 @@ async def build_env( } return { **self.config.resolved_env, + # A rollout has no interactive composer that needs suggested follow-ups. + "CODEX_ACP_PROMPT_SUGGESTIONS": "false", "CODEX_CONFIG": json.dumps(config), "CODEX_HOME": home, "DEFAULT_AUTH_REQUEST": json.dumps( From 23bedd87a780b81ff96d243906ae0957bf8f6f79 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:32:41 -0800 Subject: [PATCH 06/13] Compare branch shapes on the internal Claude Multi-Turn fixture --- tests/integration/test_multi_turn.py | 78 +++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_multi_turn.py b/tests/integration/test_multi_turn.py index ae8a2cafce..74d3687671 100644 --- a/tests/integration/test_multi_turn.py +++ b/tests/integration/test_multi_turn.py @@ -3,6 +3,8 @@ every other `dialect` a harness config can select. A turn the harness re-sends differently, beyond what its `replay_view` declares, forks the trace.""" +from collections import defaultdict + import pytest from grid import RUNTIMES, assert_one_branch, dialect_grid, grid, mark @@ -20,15 +22,79 @@ @pytest.mark.parametrize("harness,harness_runtime", MULTI_TURN, indirect=True) async def test_multi_turn(run, harness, harness_runtime): - (episode,) = await run( + episodes = await run( "echo-agentic", harness=harness, runtime=harness_runtime, max_turns=10, max_tokens=8192, + n=12 + if harness.id == "claude-code" and harness_runtime["type"] == "prime" + else 1, ) - (trace,) = episode.traces - assert_one_branch(trace, harness, forks=("openclaw",)) - assert trace.ok, [error.message for error in trace.errors] - assert trace.num_turns >= 1 # ran a command, then finished - assert trace.reward == 1.0 + for episode in episodes: + (trace,) = episode.traces + if harness.id == "claude-code" and trace.num_branches > 1: + siblings = defaultdict(list) + for i, node in enumerate(trace.nodes): + siblings[node.parent].append(i) + for parent, indices in siblings.items(): + for j in indices[1:]: + a, b = trace.nodes[indices[0]], trace.nodes[j] + print( + "CLAUDE FORK", + parent, + a.sampled, + b.sampled, + a.message.role, + b.message.role, + len(a.tools), + len(b.tools), + a.tools == b.tools, + ) + for key in ( + "content", + "reasoning_content", + "provider_state", + "tool_calls", + "tool_call_id", + ): + left = getattr(a.message, key, None) + right = getattr(b.message, key, None) + if left != right: + print( + "CLAUDE DIFFERENCE", + key, + type(left).__name__, + type(right).__name__, + len(left or []), + len(right or []), + ) + if isinstance(left, str) and isinstance(right, str): + print( + "CLAUDE WHITESPACE", left.strip() == right.strip() + ) + if a.message.role == b.message.role == "system": + for label in ( + "command prefix", + "command injection", + "file paths", + "title", + "summariz", + "safety classifier", + "bash commands", + "permission", + "skill", + ): + print( + "CLAUDE SYSTEM CATEGORY", + label, + label in str(a.message.content).lower(), + label in str(b.message.content).lower(), + ) + for episode in episodes: + (trace,) = episode.traces + assert_one_branch(trace, harness, forks=("openclaw",)) + assert trace.ok, [error.message for error in trace.errors] + assert trace.num_turns >= 1 # ran a command, then finished + assert trace.reward == 1.0 From 18fbbcfe964a82ac02c5cd3f825994ea9ce387eb Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:39:51 -0800 Subject: [PATCH 07/13] Install from the Codex package root and distinguish Claude replay shapes --- tests/integration/test_multi_turn.py | 36 ++++++++++++++++++++++++- verifiers/v1/harnesses/codex/harness.py | 5 ++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_multi_turn.py b/tests/integration/test_multi_turn.py index 74d3687671..33dfa6f5c8 100644 --- a/tests/integration/test_multi_turn.py +++ b/tests/integration/test_multi_turn.py @@ -3,6 +3,7 @@ every other `dialect` a harness config can select. A turn the harness re-sends differently, beyond what its `replay_view` declares, forks the trace.""" +import json from collections import defaultdict import pytest @@ -28,7 +29,7 @@ async def test_multi_turn(run, harness, harness_runtime): runtime=harness_runtime, max_turns=10, max_tokens=8192, - n=12 + n=16 if harness.id == "claude-code" and harness_runtime["type"] == "prime" else 1, ) @@ -92,6 +93,39 @@ async def test_multi_turn(run, harness, harness_runtime): label in str(a.message.content).lower(), label in str(b.message.content).lower(), ) + if a.message.role == b.message.role == "assistant": + for first, replay in zip( + a.message.tool_calls or [], b.message.tool_calls or [] + ): + print( + "CLAUDE CALL", + first.name == "Bash", + first.name == replay.name, + first.id == replay.id, + first.namespace == replay.namespace, + ) + x, y = ( + json.loads(first.arguments), + json.loads(replay.arguments), + ) + if isinstance(x, dict) and isinstance(y, dict): + for k in ( + "command", + "description", + "timeout", + "run_in_background", + "file_path", + "content", + ): + if x.get(k) != y.get(k): + print( + "CLAUDE PARAM", + k, + type(x.get(k)).__name__, + type(y.get(k)).__name__, + str(x.get(k)).strip() + == str(y.get(k)).strip(), + ) for episode in episodes: (trace,) = episode.traces assert_one_branch(trace, harness, forks=("openclaw",)) diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 79bcec4026..43852deec4 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -31,9 +31,10 @@ rm -f {ready} # Resolve the adapter's bundled CLI to the harness pin, including across minor releases. mkdir -p {packages} +cd {packages} printf '{"overrides":{"@agentclientprotocol/codex-acp":{"@openai/codex":"%s"}}}\n' \ - "$AGENT_CODEX_VERSION" > {packages}/package.json -npm install --prefix {packages} --ignore-scripts --no-audit --no-fund \ + "$AGENT_CODEX_VERSION" > package.json +npm install --ignore-scripts --no-audit --no-fund \ --omit=dev \ "@agentclientprotocol/codex-acp@$AGENT_CODEX_ACP_VERSION" \ "@openai/codex@$AGENT_CODEX_VERSION" >/dev/null From 3863eae7352488c582b2a47372e9c26947634e83 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:49:50 -0800 Subject: [PATCH 08/13] Match Claude replays that drop standalone blank text blocks --- tests/integration/fixtures/echo_agentic.py | 3 +- tests/integration/grid.py | 21 +++- tests/integration/test_multi_turn.py | 112 +----------------- tests/unit/test_graph.py | 47 ++++++++ verifiers/v1/dialects/anthropic.py | 11 +- verifiers/v1/harnesses/claude_code/harness.py | 7 +- verifiers/v1/harnesses/utils/replay.py | 19 +++ verifiers/v1/types.py | 4 + 8 files changed, 109 insertions(+), 115 deletions(-) diff --git a/tests/integration/fixtures/echo_agentic.py b/tests/integration/fixtures/echo_agentic.py index a0402ec63a..35ebc057cf 100644 --- a/tests/integration/fixtures/echo_agentic.py +++ b/tests/integration/fixtures/echo_agentic.py @@ -53,7 +53,8 @@ def load(self) -> list[EchoAgenticTask]: idx=0, prompt=( f"Use the bash tool to write exactly the text '{phrase}' to a file " - f"named {TARGET} in the current directory, then finish." + f"named {TARGET} in the current directory. Use only the arguments " + "required to run the command, then finish." ), system_prompt=SYSTEM, answer=phrase, diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 03154d421f..d7ace173e1 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -94,8 +94,25 @@ def assert_one_branch( assert set(forks) <= FORKS.keys(), ( f"no FORKS reason for {set(forks) - FORKS.keys()}" ) - if harness.id not in forks: - assert trace.num_branches == 1, f"{trace.num_branches} branches" + if harness.id in forks or trace.num_branches == 1: + return + first_child: dict[int | None, int] = {} + for i, node in enumerate(trace.nodes): + first = first_child.setdefault(node.parent, i) + if first != i: + sibling = trace.nodes[first] + a, b = sibling.message, node.message + fields = type(a).model_fields.keys() | type(b).model_fields.keys() + changed = sorted( + k for k in fields if getattr(a, k, None) != getattr(b, k, None) + ) + if sibling.tools != node.tools: + changed.append("tools") + pytest.fail( + f"{trace.num_branches} branches at parent {node.parent}, " + f"nodes {first} ({a.role}) and {i} ({b.role}), differing fields {changed}" + ) + pytest.fail(f"{trace.num_branches} branches") def agent_seats(taskset: str, env_id: str = "") -> list[str]: diff --git a/tests/integration/test_multi_turn.py b/tests/integration/test_multi_turn.py index 33dfa6f5c8..ae8a2cafce 100644 --- a/tests/integration/test_multi_turn.py +++ b/tests/integration/test_multi_turn.py @@ -3,9 +3,6 @@ every other `dialect` a harness config can select. A turn the harness re-sends differently, beyond what its `replay_view` declares, forks the trace.""" -import json -from collections import defaultdict - import pytest from grid import RUNTIMES, assert_one_branch, dialect_grid, grid, mark @@ -23,112 +20,15 @@ @pytest.mark.parametrize("harness,harness_runtime", MULTI_TURN, indirect=True) async def test_multi_turn(run, harness, harness_runtime): - episodes = await run( + (episode,) = await run( "echo-agentic", harness=harness, runtime=harness_runtime, max_turns=10, max_tokens=8192, - n=16 - if harness.id == "claude-code" and harness_runtime["type"] == "prime" - else 1, ) - for episode in episodes: - (trace,) = episode.traces - if harness.id == "claude-code" and trace.num_branches > 1: - siblings = defaultdict(list) - for i, node in enumerate(trace.nodes): - siblings[node.parent].append(i) - for parent, indices in siblings.items(): - for j in indices[1:]: - a, b = trace.nodes[indices[0]], trace.nodes[j] - print( - "CLAUDE FORK", - parent, - a.sampled, - b.sampled, - a.message.role, - b.message.role, - len(a.tools), - len(b.tools), - a.tools == b.tools, - ) - for key in ( - "content", - "reasoning_content", - "provider_state", - "tool_calls", - "tool_call_id", - ): - left = getattr(a.message, key, None) - right = getattr(b.message, key, None) - if left != right: - print( - "CLAUDE DIFFERENCE", - key, - type(left).__name__, - type(right).__name__, - len(left or []), - len(right or []), - ) - if isinstance(left, str) and isinstance(right, str): - print( - "CLAUDE WHITESPACE", left.strip() == right.strip() - ) - if a.message.role == b.message.role == "system": - for label in ( - "command prefix", - "command injection", - "file paths", - "title", - "summariz", - "safety classifier", - "bash commands", - "permission", - "skill", - ): - print( - "CLAUDE SYSTEM CATEGORY", - label, - label in str(a.message.content).lower(), - label in str(b.message.content).lower(), - ) - if a.message.role == b.message.role == "assistant": - for first, replay in zip( - a.message.tool_calls or [], b.message.tool_calls or [] - ): - print( - "CLAUDE CALL", - first.name == "Bash", - first.name == replay.name, - first.id == replay.id, - first.namespace == replay.namespace, - ) - x, y = ( - json.loads(first.arguments), - json.loads(replay.arguments), - ) - if isinstance(x, dict) and isinstance(y, dict): - for k in ( - "command", - "description", - "timeout", - "run_in_background", - "file_path", - "content", - ): - if x.get(k) != y.get(k): - print( - "CLAUDE PARAM", - k, - type(x.get(k)).__name__, - type(y.get(k)).__name__, - str(x.get(k)).strip() - == str(y.get(k)).strip(), - ) - for episode in episodes: - (trace,) = episode.traces - assert_one_branch(trace, harness, forks=("openclaw",)) - assert trace.ok, [error.message for error in trace.errors] - assert trace.num_turns >= 1 # ran a command, then finished - assert trace.reward == 1.0 + (trace,) = episode.traces + assert_one_branch(trace, harness, forks=("openclaw",)) + assert trace.ok, [error.message for error in trace.errors] + assert trace.num_turns >= 1 # ran a command, then finished + assert trace.reward == 1.0 diff --git a/tests/unit/test_graph.py b/tests/unit/test_graph.py index 4df44699ed..92c380e5ab 100644 --- a/tests/unit/test_graph.py +++ b/tests/unit/test_graph.py @@ -5,6 +5,11 @@ import verifiers.v1 as vf from verifiers.v1 import graph +from verifiers.v1.dialects.anthropic import AnthropicDialect +from verifiers.v1.harnesses.claude_code.harness import ( + ClaudeCodeHarness, + ClaudeCodeHarnessConfig, +) from verifiers.v1.harnesses.utils.replay import drop_blank_text, strip_text from verifiers.v1.types import TurnTokens @@ -321,6 +326,48 @@ def test_replay_view_matches_cosmetically_rewritten_turns(view, sampled, replaye assert trace.num_branches == 1 +@pytest.mark.parametrize( + "blocks,branches", + [(["One ", "\n\n", " two"], 1), (["One \n\n two"], 2)], +) +def test_claude_replay_only_drops_independently_blank_text_blocks(blocks, branches): + dialect = AnthropicDialect() + harness = ClaudeCodeHarness(ClaudeCodeHarnessConfig()) + response = dialect.validate_response( + { + "type": "message", + "id": "msg", + "model": "test", + "role": "assistant", + "content": [{"type": "text", "text": block} for block in blocks], + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + sampled = dialect.parse_response(response).message + assert sampled.content == "One \n\n two" + (replayed,) = dialect.parse_request( + { + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "One "}, + {"type": "text", "text": " two"}, + ], + } + ] + } + ).messages + fields = {"content", "text_block_offsets"} + trace = _replayed_trace( + harness.replay_view, + sampled.model_dump(include=fields), + replayed.model_dump(include=fields), + dialect="anthropic_messages", + ) + assert trace.num_branches == branches + + @pytest.mark.parametrize( "dialect,branches", [("chat_completions", 1), ("responses", 2)] ) diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 0939f00736..520a339e7c 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -10,6 +10,7 @@ import re from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from itertools import accumulate from anthropic.types import Message as AnthropicMessage from anthropic.types import Usage as AnthropicUsage @@ -241,6 +242,10 @@ def content_to_wire(content) -> str | list[dict]: return blocks +def _text_block_offsets(blocks: list[str]) -> list[int] | None: + return list(accumulate(map(len, blocks[:-1]))) or None + + def parse_messages(body: dict) -> Messages: """The request's top-level `system` + `messages` -> typed messages. Assistant turns fold their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's @@ -260,7 +265,7 @@ def parse_messages(body: dict) -> Messages: block for block in blocks if block["type"] not in ("text", "tool_use") ] state.sort(key=lambda block: THINKING_ORDER.get(block["type"], 2)) - text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text") + text = [b.get("text", "") for b in blocks if b.get("type") == "text"] reasoning = "".join( b.get("thinking", "") for b in blocks if b.get("type") == "thinking" ) @@ -276,7 +281,8 @@ def parse_messages(body: dict) -> Messages: ] prompt.append( AssistantMessage( - content=text or None, + content="".join(text) or None, + text_block_offsets=_text_block_offsets(text), reasoning_content=reasoning or None, tool_calls=calls or None, provider_state=state or None, @@ -351,6 +357,7 @@ def response_from_wire(message: AnthropicMessage) -> Response: model=message.model, message=AssistantMessage( content="".join(content) or None, + text_block_offsets=_text_block_offsets(content), reasoning_content="".join(reasoning) or None, tool_calls=calls or None, provider_state=state or None, diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 6b0cf39cee..202d876e2a 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -8,7 +8,7 @@ from verifiers.v1.configs.harness import DialectName, HarnessConfig, PinnedVersion from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir -from verifiers.v1.harnesses.utils.replay import drop_blank_text +from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -46,10 +46,9 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): SUPPORTS_SKILLS = True SUPPORTS_TOOL_INTERCEPTION = True - # Claude Code drops whitespace-only text blocks on replay. Closed source: observed - # with 2.1.314, recheck on version bumps. + # Claude Code 2.1.327 filters each whitespace-only text block on replay. def replay_view(self, message: Message, dialect: DialectName | None) -> Message: - return drop_blank_text(message) + return drop_blank_text_blocks(message) async def setup(self, runtime: Runtime) -> None: await ensure_node(runtime) diff --git a/verifiers/v1/harnesses/utils/replay.py b/verifiers/v1/harnesses/utils/replay.py index 8f1886f4f0..7810b75850 100644 --- a/verifiers/v1/harnesses/utils/replay.py +++ b/verifiers/v1/harnesses/utils/replay.py @@ -2,6 +2,8 @@ `Harness.replay_view` overrides. Each function is idempotent and only normalizes whitespace, so the replayed and the sampled form of a turn map to the same view.""" +from itertools import pairwise + from verifiers.v1.types import AssistantMessage, Message @@ -17,6 +19,23 @@ def drop_blank_text(message: Message) -> Message: return message +def drop_blank_text_blocks(message: Message) -> Message: + """Drop independently blank blocks while preserving whitespace inside other blocks.""" + if not isinstance(message, AssistantMessage) or not message.text_block_offsets: + return drop_blank_text(message) + text = message.content or "" + offsets = (0, *message.text_block_offsets, len(text)) + if any(start > end for start, end in pairwise(offsets)): + return drop_blank_text(message) + blocks = (text[start:end] for start, end in pairwise(offsets)) + return message.model_copy( + update={ + "content": "".join(block for block in blocks if block.strip()) or None, + "text_block_offsets": None, + } + ) + + def strip_text(message: Message) -> Message: """Clients that store assistant text stripped, and pad a missing reasoning field (or Anthropic thinking block) with a single space on replay.""" diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index 9961ef4ea1..d67782c39a 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -136,6 +136,10 @@ class ToolCall(BaseModel): class AssistantMessage(BaseModel): role: Literal["assistant"] = "assistant" content: str | None = None + text_block_offsets: list[int] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + """Character offsets between native text blocks concatenated into content.""" reasoning_content: str | None = None tool_calls: list[ToolCall] | None = None provider_state: list[dict[str, Any]] | None = None From faffba2de3195e44303db8f21bf11c5afbd03d2c Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:59:18 -0800 Subject: [PATCH 09/13] Use the same replay key for tool results and graph matching --- tests/unit/test_agent.py | 42 +++++++++++++++++++++++++ verifiers/v1/graph.py | 12 ++++--- verifiers/v1/interception/server.py | 2 +- verifiers/v1/session.py | 49 ++++++++++++++++++++--------- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..548f4fb4aa 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -8,8 +8,11 @@ import verifiers.v1 as vf from verifiers.v1.agent import Interaction +from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute from verifiers.v1.graph import MessageNode +from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.rollout import Rollout, RolloutTimeouts +from verifiers.v1.session import RolloutSession from verifiers.v1.types import AssistantMessage @@ -33,6 +36,45 @@ def make_trace() -> vf.Trace: ) +async def test_blocked_tool_result_survives_an_assistant_replay(): + def replace_result(request: vf.Request) -> vf.Request | None: + tool = request.messages[-1] + if isinstance(tool, vf.ToolMessage) and tool.content == "": + tool.content = "Replacement result" + return request + return None + + trace = make_trace() + trace._replay_view = lambda message, dialect: ( + drop_blank_text_blocks(message) if dialect == "anthropic_messages" else message + ) + sampled = AssistantMessage( + content="Ready\n", + text_block_offsets=[5], + tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")], + ) + trace.nodes.append(MessageNode(message=sampled, sampled=True)) + session = RolloutSession( + ModelRoute(ModelContext(model="test", client=EvalClientConfig())), + trace, + request_interceptors=[replace_result], + gates_tools=True, + ) + await session.gate_tool_calls(0, "anthropic_messages") + assert (await session.decide_tool("call"))["action"] == "deny" + replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) + request = vf.Request( + messages=[ + replay, + vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), + ] + ) + rewritten, _, _ = await session.rewrite_request( + request, dialect="anthropic_messages" + ) + assert rewritten.messages[-1].content == "Replacement result" + + def make_rollout(trace: vf.Trace, session, time_remaining=None): run = FailingSegmentRollout() run.trace = trace diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index e47fd68e0c..9eb5f16c7c 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -414,7 +414,9 @@ def _tools_hash(tools: list[Tool] | None) -> str: return hashlib.blake2b(data, digest_size=16).hexdigest() -def _message_key(trace: Trace, message: Message, dialect: DialectName | None) -> str: +def replay_message_hash( + trace: Trace, message: Message, dialect: DialectName | None +) -> str: """`message_hash` of the message as the trace's harness replays it over `dialect`, so a turn the harness re-sends with cosmetic rewrites still matches the node it came from.""" @@ -433,7 +435,7 @@ def _node_key( return ( parent, _tools_hash(tools) if parent is None else None, - _message_key(trace, message, dialect), + replay_message_hash(trace, message, dialect), ) @@ -472,12 +474,12 @@ def message_prefix_len( parents: list[int | None] = [None] matched = 0 for message in prompt: - key = _message_key(trace, message, dialect) + key = replay_message_hash(trace, message, dialect) parents = [ node_id for parent in parents for node_id in children.get(parent, []) - if _message_key(trace, trace.nodes[node_id].message, dialect) == key + if replay_message_hash(trace, trace.nodes[node_id].message, dialect) == key ] if not parents: break @@ -743,7 +745,7 @@ def prepare_turn( if len(children) == 1 and trace.nodes[children[0]].message == msg: existing = children[0] if existing is None: - message_key = _message_key(trace, msg, dialect) + message_key = replay_message_hash(trace, msg, dialect) existing = idx.get( (parent, tools_hash if parent is None else None, message_key) ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 95b831a1c2..9f44e9550e 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1013,7 +1013,7 @@ async def sample() -> web.Response: session.consume_prepared(turn.tail) session.trace.response_rewrites.extend(response_rewrites) if stopped is None: - stopped = await session.gate_tool_calls(node) + stopped = await session.gate_tool_calls(node, dialect.name) if stopped is not None: session.trace.stop(stopped) return _error_response( diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index b06d6f8a55..6103439b71 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -217,6 +217,8 @@ class RolloutSession: the trace holds the pinned version, and a differing copy would fork it at that node.""" gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" + tool_dialects: dict[int, DialectName] = field(default_factory=dict) + """Wire dialect of each issuing node, used when its harness asks the tool gate.""" prepared_users: Counter[str] = field(default_factory=Counter) last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and @@ -239,7 +241,9 @@ def ctx(self) -> ModelContext: def stopped(self) -> bool: return self.trace.stop_condition is not None - def tool_result_keys(self, request: Request) -> dict[int, tuple[str, str]]: + def tool_result_keys( + self, request: Request, dialect: DialectName | None = None + ) -> dict[int, tuple[str, str]]: """Identify results by their issuing conversation prefix and call id. Canonical pinned history gives native and rewritten replays the same occurrence keys.""" prefix = hashlib.blake2b(digest_size=16) @@ -250,7 +254,9 @@ def tool_result_keys(self, request: Request) -> dict[int, tuple[str, str]]: key = calls[message.tool_call_id] keys[position] = key message = self.pinned_tool_results.get(key, message) - prefix.update(graph.message_hash(message).encode()) + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) if isinstance(message, AssistantMessage): calls.update( (call.id, (prefix.hexdigest(), call.id)) @@ -258,13 +264,17 @@ def tool_result_keys(self, request: Request) -> dict[int, tuple[str, str]]: ) return keys - def pin_tool_results(self, request: Request) -> Request: + def pin_tool_results( + self, request: Request, dialect: DialectName | None = None + ) -> Request: """Re-impose the pinned results on the harness's request. A result for a blocked call the harness never gated means the call ran anyway: the transcript would no longer describe the sandbox, so that fails the rollout instead of being papered over.""" + if not self.pinned_tool_results: + return request messages = list(request.messages) changed = False - for position, key in self.tool_result_keys(request).items(): + for position, key in self.tool_result_keys(request, dialect).items(): pinned = self.pinned_tool_results.get(key) if pinned is None: continue @@ -287,10 +297,10 @@ async def rewrite_request( """Run typed request interceptors and stops over one canonical request. Only the uncommitted tail's user and tool messages may change; a rewritten tool result is pinned for the rest of the rollout.""" - request = self.pin_tool_results(request) - tool_keys = self.tool_result_keys(request) + request = self.pin_tool_results(request, dialect) if not self.request_interceptors and (not run_stops or not self.request_stops): return request, [], None + tool_keys = self.tool_result_keys(request, dialect) candidates: set[int] = set() tail_start = graph.message_prefix_len(self.trace, request.messages, dialect) if self.request_interceptors: @@ -399,7 +409,7 @@ async def rewrite_request( f"request interception failed: {type(error).__name__}: {error}" ) from error current = request.model_copy(update={"messages": messages}) - tool_keys = self.tool_result_keys(current) + tool_keys = self.tool_result_keys(current, dialect) for position in candidates: after = current.messages[position] if ( @@ -489,27 +499,33 @@ async def rewrite_response( ) from error return response, records, None - async def gate_tool_calls(self, node: int) -> str | None: + async def gate_tool_calls( + self, node: int, dialect: DialectName | None = None + ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a `@stop` refuses the whole response and a rewrite becomes that call's verdict — the harness's gate denies it and the model sees the rewritten result instead. Returns the name of the stop that fired, if any.""" assistant = self.trace.nodes[node].message - if not (self.request_interceptors or self.request_stops) or not ( - isinstance(assistant, AssistantMessage) and assistant.tool_calls - ): + if not (self.request_interceptors or self.request_stops): + return None + if dialect is not None: + self.tool_dialects[node] = dialect + if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: return None branch = graph.path(self.trace, node) for call in assistant.tool_calls: probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) request = Request(messages=[*branch, probe], tools=self.trace.tools or None) - key = self.tool_result_keys(request)[len(branch)] + key = self.tool_result_keys(request, dialect)[len(branch)] if key in self.tool_decisions: raise HarnessError( f"tool call id {call.id!r} was reused in the same assistant turn" ) - request, records, stopped = await self.rewrite_request(request) + request, records, stopped = await self.rewrite_request( + request, dialect=dialect + ) self.trace.request_rewrites.extend(records) if stopped is not None: return stopped @@ -579,7 +595,9 @@ async def decide_tool( request = Request( messages=[*branch, ToolMessage(tool_call_id=call.id, content="")] ) - key = self.tool_result_keys(request)[len(branch)] + key = self.tool_result_keys(request, self.tool_dialects.get(leaf))[ + len(branch) + ] self.gated_tools.add(key) verdict = self.tool_decisions.get(key) elif matches or not name or len(leaves) != 1: @@ -598,7 +616,8 @@ async def decide_tool( request, records, stopped = await self.rewrite_request( Request( messages=[*branch, assistant, probe], tools=self.trace.tools or None - ) + ), + dialect=self.tool_dialects.get(leaves[0][0]), ) self.trace.request_rewrites.extend(records) if stopped is not None: From d9b12d55a10846c1475f69f1dfac0b59c06754b9 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:07:02 -0800 Subject: [PATCH 10/13] Keep tool results associated with their sampled turn --- tests/unit/test_agent.py | 47 ++++++++++++++------- verifiers/v1/session.py | 91 +++++++++++++++++++++++++++------------- 2 files changed, 93 insertions(+), 45 deletions(-) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index 548f4fb4aa..e4bfcbe7a3 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -7,6 +7,7 @@ import pytest import verifiers.v1 as vf +from verifiers.v1 import graph from verifiers.v1.agent import Interaction from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute from verifiers.v1.graph import MessageNode @@ -36,11 +37,13 @@ def make_trace() -> vf.Trace: ) -async def test_blocked_tool_result_survives_an_assistant_replay(): +async def test_each_sample_keeps_its_tool_result_across_assistant_replays(): + replacements = iter(["First result", "Second result"]) + def replace_result(request: vf.Request) -> vf.Request | None: tool = request.messages[-1] if isinstance(tool, vf.ToolMessage) and tool.content == "": - tool.content = "Replacement result" + tool.content = next(replacements) return request return None @@ -53,26 +56,40 @@ def replace_result(request: vf.Request) -> vf.Request | None: text_block_offsets=[5], tool_calls=[vf.ToolCall(id="call", name="lookup", arguments="{}")], ) - trace.nodes.append(MessageNode(message=sampled, sampled=True)) session = RolloutSession( ModelRoute(ModelContext(model="test", client=EvalClientConfig())), trace, request_interceptors=[replace_result], gates_tools=True, ) - await session.gate_tool_calls(0, "anthropic_messages") - assert (await session.decide_tool("call"))["action"] == "deny" replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) - request = vf.Request( - messages=[ - replay, - vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), - ] - ) - rewritten, _, _ = await session.rewrite_request( - request, dialect="anthropic_messages" - ) - assert rewritten.messages[-1].content == "Replacement result" + prompt = [vf.UserMessage(content="go")] + + def commit(messages, assistant): + turn = graph.prepare_turn(trace, messages, dialect="anthropic_messages") + return turn.commit( + vf.Response( + id="", created=0, model="test", message=assistant, finish_reason="stop" + ) + ) + + # Two samples from one prompt replay identically and reuse a call ID. + for assistant, expected in [(sampled, "First result"), (replay, "Second result")]: + node = commit(prompt, assistant) + await session.gate_tool_calls(node, "anthropic_messages") + assert (await session.decide_tool("call"))["action"] == "deny" + request = vf.Request( + messages=[ + *prompt, + replay, + vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), + ] + ) + rewritten, _, _ = await session.rewrite_request( + request, dialect="anthropic_messages" + ) + assert rewritten.messages[-1].content == expected + commit(rewritten.messages, AssistantMessage(content="done")) def make_rollout(trace: vf.Trace, session, time_remaining=None): diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 6103439b71..f84c264aeb 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -54,6 +54,7 @@ logger = get_logger(__name__) MCPProxy = Callable[[str], AbstractAsyncContextManager[str]] +ToolResultKey = tuple[int | str, str] def hook_boundary(handler: Callable, *, allow_trace: bool) -> type: @@ -203,22 +204,22 @@ class RolloutSession: """Handler tasks currently serving this session. aiohttp does not cancel a handler when its client disconnects, so a request whose program died at teardown would keep driving the exchange (upstream call, simulator turn) — unregistering cancels these instead.""" - tool_decisions: dict[tuple[str, str], ToolMessage | None] = field( + tool_decisions: dict[ToolResultKey, ToolMessage | None] = field( default_factory=dict ) """The pre-execution verdict per proposed call occurrence: None to run it, else the result the policy put in its place — that call must not execute.""" - pinned_tool_results: dict[tuple[str, str], ToolMessage] = field( - default_factory=dict - ) + pinned_tool_results: dict[ToolResultKey, ToolMessage] = field(default_factory=dict) """The results the model must see, by conversation prefix and call id: rewrites and results it substituted for blocked calls. The harness keeps its own copy of each in its history and re-sends it forever, so every request has these re-applied before the graph is matched — the trace holds the pinned version, and a differing copy would fork it at that node.""" - gated_tools: set[tuple[str, str]] = field(default_factory=set) + gated_tools: set[ToolResultKey] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" tool_dialects: dict[int, DialectName] = field(default_factory=dict) """Wire dialect of each issuing node, used when its harness asks the tool gate.""" + tool_issuers: dict[tuple[str, str], set[int]] = field(default_factory=dict) + """Sampled nodes that issued a call ID under a replay-equivalent prefix.""" prepared_users: Counter[str] = field(default_factory=Counter) last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and @@ -242,13 +243,17 @@ def stopped(self) -> bool: return self.trace.stop_condition is not None def tool_result_keys( - self, request: Request, dialect: DialectName | None = None - ) -> dict[int, tuple[str, str]]: - """Identify results by their issuing conversation prefix and call id. Canonical - pinned history gives native and rewritten replays the same occurrence keys.""" + self, + request: Request, + dialect: DialectName | None = None, + issuing_node: int | None = None, + ) -> dict[int, ToolResultKey]: + """Use the sampled node and call ID for a model-issued tool. A harness-supplied + call outside the sampled graph uses its conversation prefix instead.""" prefix = hashlib.blake2b(digest_size=16) - calls: dict[str, tuple[str, str]] = {} - keys: dict[int, tuple[str, str]] = {} + calls: dict[str, ToolResultKey] = {} + keys: dict[int, ToolResultKey] = {} + history: Messages = [] for position, message in enumerate(request.messages): if isinstance(message, ToolMessage) and message.tool_call_id in calls: key = calls[message.tool_call_id] @@ -257,15 +262,44 @@ def tool_result_keys( prefix.update( graph.replay_message_hash(self.trace, message, dialect).encode() ) - if isinstance(message, AssistantMessage): - calls.update( - (call.id, (prefix.hexdigest(), call.id)) - for call in message.tool_calls or [] - ) + history.append(message) + if isinstance(message, AssistantMessage) and message.tool_calls: + prefix_hash = prefix.hexdigest() + for call in message.tool_calls: + alias = (prefix_hash, call.id) + if ( + issuing_node is not None + and position == len(request.messages) - 2 + ): + self.tool_issuers.setdefault(alias, set()).add(issuing_node) + calls[call.id] = (issuing_node, call.id) + elif issuers := self.tool_issuers.get(alias): + if len(issuers) == 1: + issuer = next(iter(issuers)) + else: + # Resolve replay-equivalent sibling samples just as the graph + # resolves the incoming model turn, including its root tools. + matched = graph.prepare_turn( + self.trace, history, request.tools, dialect + ).prefix_node_ids + if ( + len(matched) != len(history) + or matched[-1] not in issuers + ): + raise HarnessError( + f"Ambiguous issuing turn for tool call {call.id!r}" + ) + issuer = matched[-1] + calls[call.id] = (issuer, call.id) + else: + calls[call.id] = alias return keys def pin_tool_results( - self, request: Request, dialect: DialectName | None = None + self, + request: Request, + dialect: DialectName | None = None, + issuing_node: int | None = None, ) -> Request: """Re-impose the pinned results on the harness's request. A result for a blocked call the harness never gated means the call ran anyway: the transcript would no longer @@ -274,7 +308,9 @@ def pin_tool_results( return request messages = list(request.messages) changed = False - for position, key in self.tool_result_keys(request, dialect).items(): + for position, key in self.tool_result_keys( + request, dialect, issuing_node + ).items(): pinned = self.pinned_tool_results.get(key) if pinned is None: continue @@ -293,14 +329,15 @@ async def rewrite_request( *, run_stops: bool = True, dialect: DialectName | None = None, + issuing_node: int | None = None, ) -> tuple[Request, list[InterceptRecord], str | None]: """Run typed request interceptors and stops over one canonical request. Only the uncommitted tail's user and tool messages may change; a rewritten tool result is pinned for the rest of the rollout.""" - request = self.pin_tool_results(request, dialect) + request = self.pin_tool_results(request, dialect, issuing_node) if not self.request_interceptors and (not run_stops or not self.request_stops): return request, [], None - tool_keys = self.tool_result_keys(request, dialect) + tool_keys = self.tool_result_keys(request, dialect, issuing_node) candidates: set[int] = set() tail_start = graph.message_prefix_len(self.trace, request.messages, dialect) if self.request_interceptors: @@ -409,7 +446,7 @@ async def rewrite_request( f"request interception failed: {type(error).__name__}: {error}" ) from error current = request.model_copy(update={"messages": messages}) - tool_keys = self.tool_result_keys(current, dialect) + tool_keys = self.tool_result_keys(current, dialect, issuing_node) for position in candidates: after = current.messages[position] if ( @@ -518,13 +555,13 @@ async def gate_tool_calls( for call in assistant.tool_calls: probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) request = Request(messages=[*branch, probe], tools=self.trace.tools or None) - key = self.tool_result_keys(request, dialect)[len(branch)] + key = (node, call.id) if key in self.tool_decisions: raise HarnessError( f"tool call id {call.id!r} was reused in the same assistant turn" ) request, records, stopped = await self.rewrite_request( - request, dialect=dialect + request, dialect=dialect, issuing_node=node ) self.trace.request_rewrites.extend(records) if stopped is not None: @@ -591,13 +628,7 @@ async def decide_tool( ] or matches if len(matches) == 1: leaf, call = matches[0] - branch = graph.path(self.trace, leaf) - request = Request( - messages=[*branch, ToolMessage(tool_call_id=call.id, content="")] - ) - key = self.tool_result_keys(request, self.tool_dialects.get(leaf))[ - len(branch) - ] + key = (leaf, call.id) self.gated_tools.add(key) verdict = self.tool_decisions.get(key) elif matches or not name or len(leaves) != 1: From 7507434cabba54a1d7195af91ff26abdf23ef2be Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:15:55 -0800 Subject: [PATCH 11/13] Resolve gate history against its actual branch ancestry --- tests/unit/test_agent.py | 19 ++++++++++++--- verifiers/v1/graph.py | 17 +++++++++----- verifiers/v1/session.py | 51 +++++++++++++++++++++++++--------------- verifiers/v1/trace.py | 7 +----- 4 files changed, 60 insertions(+), 34 deletions(-) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index e4bfcbe7a3..2245622cc3 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -38,7 +38,7 @@ def make_trace() -> vf.Trace: async def test_each_sample_keeps_its_tool_result_across_assistant_replays(): - replacements = iter(["First result", "Second result"]) + replacements = iter(["First result", "Second result", "Follow-up result"]) def replace_result(request: vf.Request) -> vf.Request | None: tool = request.messages[-1] @@ -65,8 +65,10 @@ def replace_result(request: vf.Request) -> vf.Request | None: replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) prompt = [vf.UserMessage(content="go")] - def commit(messages, assistant): - turn = graph.prepare_turn(trace, messages, dialect="anthropic_messages") + def commit(messages, assistant, tools=None): + turn = graph.prepare_turn( + trace, messages, tools=tools, dialect="anthropic_messages" + ) return turn.commit( vf.Response( id="", created=0, model="test", message=assistant, finish_reason="stop" @@ -91,6 +93,17 @@ def commit(messages, assistant): assert rewritten.messages[-1].content == expected commit(rewritten.messages, AssistantMessage(content="done")) + # An unrelated branch adds a tool to the trace-wide catalog. The next gate + # still knows exactly which sampled ancestors issued its historical calls. + commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")]) + follow_up = AssistantMessage( + tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")] + ) + node = commit(rewritten.messages, follow_up) + await session.gate_tool_calls(node, "anthropic_messages") + verdict = await session.decide_tool("next") + assert verdict["message"]["content"] == "Follow-up result" + def make_rollout(trace: vf.Trace, session, time_remaining=None): run = FailingSegmentRollout() diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 9eb5f16c7c..90c6c897d0 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -1142,12 +1142,17 @@ def leaves(trace: Trace) -> list[int]: return [i for i in range(len(trace.nodes)) if i not in has_child] -def path(trace: Trace, node: int) -> list[Message]: - """The conversation ending at `node`: its root-to-node messages.""" - messages: list[Message] = [] +def path_node_ids(trace: Trace, node: int) -> list[int]: + """The physical ancestry from the root through the given node.""" + nodes: list[int] = [] current: int | None = node while current is not None: - messages.append(trace.nodes[current].message) + nodes.append(current) current = trace.nodes[current].parent - messages.reverse() - return messages + nodes.reverse() + return nodes + + +def path(trace: Trace, node: int) -> list[Message]: + """The conversation ending at `node`: its root-to-node messages.""" + return [trace.nodes[i].message for i in path_node_ids(trace, node)] diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index f84c264aeb..ddac3e61c1 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -19,7 +19,7 @@ import inspect import json from collections import Counter -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from contextlib import AbstractAsyncContextManager from dataclasses import dataclass, field, replace from functools import cached_property @@ -246,7 +246,7 @@ def tool_result_keys( self, request: Request, dialect: DialectName | None = None, - issuing_node: int | None = None, + known_path: Sequence[int] = (), ) -> dict[int, ToolResultKey]: """Use the sampled node and call ID for a model-issued tool. A harness-supplied call outside the sampled graph uses its conversation prefix instead.""" @@ -268,11 +268,12 @@ def tool_result_keys( for call in message.tool_calls: alias = (prefix_hash, call.id) if ( - issuing_node is not None - and position == len(request.messages) - 2 + position < len(known_path) + and self.trace.nodes[known_path[position]].sampled ): - self.tool_issuers.setdefault(alias, set()).add(issuing_node) - calls[call.id] = (issuing_node, call.id) + issuer = known_path[position] + self.tool_issuers.setdefault(alias, set()).add(issuer) + calls[call.id] = (issuer, call.id) elif issuers := self.tool_issuers.get(alias): if len(issuers) == 1: issuer = next(iter(issuers)) @@ -299,7 +300,7 @@ def pin_tool_results( self, request: Request, dialect: DialectName | None = None, - issuing_node: int | None = None, + known_path: Sequence[int] = (), ) -> Request: """Re-impose the pinned results on the harness's request. A result for a blocked call the harness never gated means the call ran anyway: the transcript would no longer @@ -309,7 +310,7 @@ def pin_tool_results( messages = list(request.messages) changed = False for position, key in self.tool_result_keys( - request, dialect, issuing_node + request, dialect, known_path ).items(): pinned = self.pinned_tool_results.get(key) if pinned is None: @@ -329,15 +330,15 @@ async def rewrite_request( *, run_stops: bool = True, dialect: DialectName | None = None, - issuing_node: int | None = None, + known_path: Sequence[int] = (), ) -> tuple[Request, list[InterceptRecord], str | None]: """Run typed request interceptors and stops over one canonical request. Only the uncommitted tail's user and tool messages may change; a rewritten tool result is pinned for the rest of the rollout.""" - request = self.pin_tool_results(request, dialect, issuing_node) + request = self.pin_tool_results(request, dialect, known_path) if not self.request_interceptors and (not run_stops or not self.request_stops): return request, [], None - tool_keys = self.tool_result_keys(request, dialect, issuing_node) + tool_keys = self.tool_result_keys(request, dialect, known_path) candidates: set[int] = set() tail_start = graph.message_prefix_len(self.trace, request.messages, dialect) if self.request_interceptors: @@ -446,7 +447,7 @@ async def rewrite_request( f"request interception failed: {type(error).__name__}: {error}" ) from error current = request.model_copy(update={"messages": messages}) - tool_keys = self.tool_result_keys(current, dialect, issuing_node) + tool_keys = self.tool_result_keys(current, dialect, known_path) for position in candidates: after = current.messages[position] if ( @@ -469,9 +470,16 @@ async def prepare_users( self, request: Request ) -> tuple[Request, list[InterceptRecord]]: """Intercept caller-owned user turns before the harness stores them.""" - branch = self.trace.messages + known_path = ( + graph.path_node_ids(self.trace, len(self.trace.nodes) - 1) + if self.trace.nodes + else [] + ) + branch = [self.trace.nodes[i].message for i in known_path] rewritten, records, _ = await self.rewrite_request( - Request(messages=[*branch, *request.messages]), run_stops=False + Request(messages=[*branch, *request.messages]), + run_stops=False, + known_path=known_path, ) tail = rewritten.messages[len(branch) :] self.prepared_users.update( @@ -551,17 +559,19 @@ async def gate_tool_calls( self.tool_dialects[node] = dialect if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: return None - branch = graph.path(self.trace, node) + known_path = graph.path_node_ids(self.trace, node) + branch = [self.trace.nodes[i].message for i in known_path] + tools = self.trace.nodes[known_path[0]].tools for call in assistant.tool_calls: probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) - request = Request(messages=[*branch, probe], tools=self.trace.tools or None) + request = Request(messages=[*branch, probe], tools=tools or None) key = (node, call.id) if key in self.tool_decisions: raise HarnessError( f"tool call id {call.id!r} was reused in the same assistant turn" ) request, records, stopped = await self.rewrite_request( - request, dialect=dialect, issuing_node=node + request, dialect=dialect, known_path=known_path ) self.trace.request_rewrites.extend(records) if stopped is not None: @@ -637,7 +647,8 @@ async def decide_tool( f"{len(matches)} proposed calls" ) else: - *branch, assistant = graph.path(self.trace, leaves[0][0]) + known_path = graph.path_node_ids(self.trace, leaves[0][0]) + *branch, assistant = [self.trace.nodes[i].message for i in known_path] assert isinstance(assistant, AssistantMessage) call = ToolCall(id=tool_call_id, name=name, arguments=json.dumps(arguments)) assistant = assistant.model_copy( @@ -646,9 +657,11 @@ async def decide_tool( probe = ToolMessage(tool_call_id=call.id, content="", name=name) request, records, stopped = await self.rewrite_request( Request( - messages=[*branch, assistant, probe], tools=self.trace.tools or None + messages=[*branch, assistant, probe], + tools=self.trace.nodes[known_path[0]].tools or None, ), dialect=self.tool_dialects.get(leaves[0][0]), + known_path=known_path[:-1], ) self.trace.request_rewrites.extend(records) if stopped is not None: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 6bcbe22179..e695a3aaca 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -742,12 +742,7 @@ def branches(self) -> list[Branch]: } branches: list[Branch] = [] for i, leaf in enumerate(graph.leaves(self)): - path: list[int] = [] - nid: int | None = leaf - while nid is not None: - path.append(nid) - nid = self.nodes[nid].parent - path.reverse() + path = graph.path_node_ids(self, leaf) is_compaction_attempt = any( link.type == "compaction_attempt" for link in self.nodes[leaf].semantic_parents From edba265dba1d8d152e226a077f2b99e18da99282 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:28:55 -0800 Subject: [PATCH 12/13] Resolve ambiguous replayed tool receipts using the native catalog --- tests/unit/test_agent.py | 32 +++++++++++++++++--- verifiers/v1/graph.py | 6 ++-- verifiers/v1/interception/server.py | 4 ++- verifiers/v1/session.py | 47 ++++++++++++++++------------- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index 2245622cc3..caf1fbfada 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -10,6 +10,7 @@ from verifiers.v1 import graph from verifiers.v1.agent import Interaction from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute +from verifiers.v1.errors import HarnessError from verifiers.v1.graph import MessageNode from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks from verifiers.v1.rollout import Rollout, RolloutTimeouts @@ -38,7 +39,9 @@ def make_trace() -> vf.Trace: async def test_each_sample_keeps_its_tool_result_across_assistant_replays(): - replacements = iter(["First result", "Second result", "Follow-up result"]) + replacements = iter( + ["First result", "Second result", "Follow-up result", "Third result"] + ) def replace_result(request: vf.Request) -> vf.Request | None: tool = request.messages[-1] @@ -75,17 +78,22 @@ def commit(messages, assistant, tools=None): ) ) - # Two samples from one prompt replay identically and reuse a call ID. - for assistant, expected in [(sampled, "First result"), (replay, "Second result")]: + # Different harness catalogs can yield the same model-facing root after mediation. + catalogs = [[vf.Tool(name=name)] for name in ("filtered_first", "filtered_second")] + for assistant, expected, tools in [ + (sampled, "First result", catalogs[0]), + (replay, "Second result", catalogs[1]), + ]: node = commit(prompt, assistant) - await session.gate_tool_calls(node, "anthropic_messages") + await session.gate_tool_calls(node, "anthropic_messages", tools) assert (await session.decide_tool("call"))["action"] == "deny" request = vf.Request( messages=[ *prompt, replay, vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), - ] + ], + tools=tools, ) rewritten, _, _ = await session.rewrite_request( request, dialect="anthropic_messages" @@ -93,6 +101,12 @@ def commit(messages, assistant, tools=None): assert rewritten.messages[-1].content == expected commit(rewritten.messages, AssistantMessage(content="done")) + first_request = request.model_copy(update={"tools": catalogs[0]}) + earlier, _, _ = await session.rewrite_request( + first_request, dialect="anthropic_messages" + ) + assert earlier.messages[-1].content == "First result" + # An unrelated branch adds a tool to the trace-wide catalog. The next gate # still knows exactly which sampled ancestors issued its historical calls. commit(prompt, AssistantMessage(content="other branch"), [vf.Tool(name="other")]) @@ -104,6 +118,14 @@ def commit(messages, assistant, tools=None): verdict = await session.decide_tool("next") assert verdict["message"]["content"] == "Follow-up result" + # Even the catalog is identical this time. A bare receipt cannot identify which + # of two different substitutions belongs to it. + node = commit(prompt, sampled) + await session.gate_tool_calls(node, "anthropic_messages", catalogs[0]) + assert (await session.decide_tool("call"))["message"]["content"] == "Third result" + with pytest.raises(HarnessError, match="Ambiguous issuing turn"): + await session.rewrite_request(first_request, dialect="anthropic_messages") + def make_rollout(trace: vf.Trace, session, time_remaining=None): run = FailingSegmentRollout() diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 90c6c897d0..1e0add3817 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -408,7 +408,7 @@ def add(value: str) -> None: return digest.hexdigest() -def _tools_hash(tools: list[Tool] | None) -> str: +def tool_catalog_hash(tools: list[Tool] | None) -> str: payload = [tool.model_dump(mode="json", exclude_none=True) for tool in tools or []] data = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() return hashlib.blake2b(data, digest_size=16).hexdigest() @@ -434,7 +434,7 @@ def _node_key( ) -> tuple[int | None, str | None, str]: return ( parent, - _tools_hash(tools) if parent is None else None, + tool_catalog_hash(tools) if parent is None else None, replay_message_hash(trace, message, dialect), ) @@ -723,7 +723,7 @@ def prepare_turn( is the request's wire format, whose replay view matches the prompt to stored nodes.""" idx = _head_index(trace, dialect) tools = list(tools or []) - tools_hash = _tools_hash(tools) + tools_hash = tool_catalog_hash(tools) parent: int | None = None path_len = 0 prefix_node_ids: list[int] = [] diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 9f44e9550e..009d0f69e8 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1013,7 +1013,9 @@ async def sample() -> web.Response: session.consume_prepared(turn.tail) session.trace.response_rewrites.extend(response_rewrites) if stopped is None: - stopped = await session.gate_tool_calls(node, dialect.name) + stopped = await session.gate_tool_calls( + node, dialect.name, original_request.tools + ) if stopped is not None: session.trace.stop(stopped) return _error_response( diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index ddac3e61c1..4251c2d40f 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -39,6 +39,7 @@ Messages, Request, Response, + Tool, ToolCall, ToolMessage, UserMessage, @@ -220,6 +221,8 @@ class RolloutSession: """Wire dialect of each issuing node, used when its harness asks the tool gate.""" tool_issuers: dict[tuple[str, str], set[int]] = field(default_factory=dict) """Sampled nodes that issued a call ID under a replay-equivalent prefix.""" + tool_catalogs: dict[int, str] = field(default_factory=dict) + """The harness's tool catalog when it requested each issuing turn, before mediation.""" prepared_users: Counter[str] = field(default_factory=Counter) last_request: tuple[list, Messages] = ((), []) """The last chat request's raw messages (copied: the server rewrites a body in place) and @@ -253,16 +256,32 @@ def tool_result_keys( prefix = hashlib.blake2b(digest_size=16) calls: dict[str, ToolResultKey] = {} keys: dict[int, ToolResultKey] = {} - history: Messages = [] + catalog = None for position, message in enumerate(request.messages): if isinstance(message, ToolMessage) and message.tool_call_id in calls: key = calls[message.tool_call_id] + issuer, call_id = key + if isinstance(issuer, str) and ( + candidates := self.tool_issuers.get((issuer, call_id)) + ): + if len(candidates) > 1: + if catalog is None: + catalog = graph.tool_catalog_hash(request.tools) + candidates = { + node + for node in candidates + if self.tool_catalogs[node] == catalog + } + if len(candidates) != 1: + raise HarnessError( + f"Ambiguous issuing turn for tool call {call_id!r}" + ) + key = (next(iter(candidates)), call_id) keys[position] = key message = self.pinned_tool_results.get(key, message) prefix.update( graph.replay_message_hash(self.trace, message, dialect).encode() ) - history.append(message) if isinstance(message, AssistantMessage) and message.tool_calls: prefix_hash = prefix.hexdigest() for call in message.tool_calls: @@ -274,24 +293,6 @@ def tool_result_keys( issuer = known_path[position] self.tool_issuers.setdefault(alias, set()).add(issuer) calls[call.id] = (issuer, call.id) - elif issuers := self.tool_issuers.get(alias): - if len(issuers) == 1: - issuer = next(iter(issuers)) - else: - # Resolve replay-equivalent sibling samples just as the graph - # resolves the incoming model turn, including its root tools. - matched = graph.prepare_turn( - self.trace, history, request.tools, dialect - ).prefix_node_ids - if ( - len(matched) != len(history) - or matched[-1] not in issuers - ): - raise HarnessError( - f"Ambiguous issuing turn for tool call {call.id!r}" - ) - issuer = matched[-1] - calls[call.id] = (issuer, call.id) else: calls[call.id] = alias return keys @@ -545,7 +546,10 @@ async def rewrite_response( return response, records, None async def gate_tool_calls( - self, node: int, dialect: DialectName | None = None + self, + node: int, + dialect: DialectName | None = None, + native_tools: list[Tool] | None = None, ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a @@ -559,6 +563,7 @@ async def gate_tool_calls( self.tool_dialects[node] = dialect if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: return None + self.tool_catalogs[node] = graph.tool_catalog_hash(native_tools) known_path = graph.path_node_ids(self.trace, node) branch = [self.trace.nodes[i].message for i in known_path] tools = self.trace.nodes[known_path[0]].tools From 45a365ddef0e1e033612eb7e4ed4928fc6106394 Mon Sep 17 00:00:00 2001 From: Florian Brand <46377542+xeophon@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:50:58 -0800 Subject: [PATCH 13/13] Match replayed tool results against the harness-visible prompt --- tests/unit/test_agent.py | 64 ++++++++++++++++++++++------- verifiers/v1/interception/server.py | 19 ++++++++- verifiers/v1/session.py | 15 +++++-- 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index caf1fbfada..ada7779824 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -10,6 +10,8 @@ from verifiers.v1 import graph from verifiers.v1.agent import Interaction from verifiers.v1.clients import EvalClientConfig, ModelContext, ModelRoute +from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.dialects.anthropic import AnthropicDialect from verifiers.v1.errors import HarnessError from verifiers.v1.graph import MessageNode from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks @@ -67,6 +69,21 @@ def replace_result(request: vf.Request) -> vf.Request | None: ) replay = sampled.model_copy(update={"content": "Ready", "text_block_offsets": None}) prompt = [vf.UserMessage(content="go")] + dialect = AnthropicDialect() + + def restrict(provider_tool: str): + body = { + "messages": [{"role": "user", "content": "go"}], + "tools": [ + {"name": "lookup", "input_schema": {"type": "object"}}, + {"name": provider_tool.rsplit("_", 1)[0], "type": provider_tool}, + ], + } + native = dialect.parse_request(body) + filtered, _ = dialect.mediate_external_capabilities( + body, NetworkPolicyConfig(allow=[]) + ) + return native, dialect.parse_request(filtered) def commit(messages, assistant, tools=None): turn = graph.prepare_turn( @@ -78,14 +95,18 @@ def commit(messages, assistant, tools=None): ) ) - # Different harness catalogs can yield the same model-facing root after mediation. - catalogs = [[vf.Tool(name=name)] for name in ("filtered_first", "filtered_second")] - for assistant, expected, tools in [ - (sampled, "First result", catalogs[0]), - (replay, "Second result", catalogs[1]), + # Removing a provider tool also adds a notice to the model-facing prompt. + first_native, filtered = restrict("web_search_20250305") + second_native, second_filtered = restrict("web_fetch_20250910") + assert filtered == second_filtered + assert [tool.name for tool in filtered.tools or []] == ["lookup"] + assert filtered.messages != prompt + for assistant, expected, native in [ + (sampled, "First result", first_native), + (replay, "Second result", second_native), ]: - node = commit(prompt, assistant) - await session.gate_tool_calls(node, "anthropic_messages", tools) + node = commit(filtered.messages, assistant, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", native) assert (await session.decide_tool("call"))["action"] == "deny" request = vf.Request( messages=[ @@ -93,15 +114,16 @@ def commit(messages, assistant, tools=None): replay, vf.ToolMessage(tool_call_id="call", content="Tool unavailable"), ], - tools=tools, + tools=native.tools, ) rewritten, _, _ = await session.rewrite_request( request, dialect="anthropic_messages" ) assert rewritten.messages[-1].content == expected - commit(rewritten.messages, AssistantMessage(content="done")) + model_history = [*filtered.messages, *rewritten.messages[len(prompt) :]] + commit(model_history, AssistantMessage(content="done"), filtered.tools) - first_request = request.model_copy(update={"tools": catalogs[0]}) + first_request = request.model_copy(update={"tools": first_native.tools}) earlier, _, _ = await session.rewrite_request( first_request, dialect="anthropic_messages" ) @@ -113,15 +135,29 @@ def commit(messages, assistant, tools=None): follow_up = AssistantMessage( tool_calls=[vf.ToolCall(id="next", name="lookup", arguments="{}")] ) - node = commit(rewritten.messages, follow_up) - await session.gate_tool_calls(node, "anthropic_messages") + node = commit(model_history, follow_up, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", rewritten) verdict = await session.decide_tool("next") assert verdict["message"]["content"] == "Follow-up result" + continued = request.model_copy( + update={ + "messages": [ + *request.messages, + follow_up, + vf.ToolMessage(tool_call_id="next", content="Tool unavailable"), + ] + } + ) + final, _, _ = await session.rewrite_request(continued, dialect="anthropic_messages") + assert [m.content for m in final.messages if isinstance(m, vf.ToolMessage)] == [ + "Second result", + "Follow-up result", + ] # Even the catalog is identical this time. A bare receipt cannot identify which # of two different substitutions belongs to it. - node = commit(prompt, sampled) - await session.gate_tool_calls(node, "anthropic_messages", catalogs[0]) + node = commit(filtered.messages, sampled, filtered.tools) + await session.gate_tool_calls(node, "anthropic_messages", first_native) assert (await session.decide_tool("call"))["message"]["content"] == "Third result" with pytest.raises(HarnessError, match="Ambiguous issuing turn"): await session.rewrite_request(first_request, dialect="anthropic_messages") diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 009d0f69e8..02de183224 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -67,7 +67,7 @@ from verifiers.v1.semantic import ACPInfo, extract_acp_info, routing_session_id from verifiers.v1.session import IdempotentRequest, ReplayResponse, RolloutSession from verifiers.v1.trace import Error, ModelCall, PolicyEvent, TimeSpan -from verifiers.v1.types import FinishReason, Response, Usage +from verifiers.v1.types import FinishReason, Response, ToolMessage, Usage from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.format import format_time from verifiers.v1.utils.logging import clip, get_logger, isolated @@ -881,6 +881,7 @@ async def coalesced( dialect, f"rollout stopped: {refused}", status=400 ) original_request = model_request + harness_history = original_request model_request, request_rewrites, stopped = await session.rewrite_request( model_request, dialect=dialect.name ) @@ -888,6 +889,20 @@ async def coalesced( # A pinned tool result changes the request without a fresh record. if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) + # Match future receipts against the prompt the harness will replay. + # It retains its own user messages. We reapply pinned tool results. + harness_history = original_request.model_copy( + update={ + "messages": [ + after if isinstance(after, ToolMessage) else before + for before, after in zip( + original_request.messages, + model_request.messages, + strict=True, + ) + ] + } + ) except RolloutError as error: return self._fail(session, dialect, error) except Exception as error: # noqa: BLE001 - surface task hook failures @@ -1014,7 +1029,7 @@ async def sample() -> web.Response: session.trace.response_rewrites.extend(response_rewrites) if stopped is None: stopped = await session.gate_tool_calls( - node, dialect.name, original_request.tools + node, dialect.name, harness_history ) if stopped is not None: session.trace.stop(stopped) diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index 4251c2d40f..bba474340d 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -39,7 +39,6 @@ Messages, Request, Response, - Tool, ToolCall, ToolMessage, UserMessage, @@ -291,7 +290,6 @@ def tool_result_keys( and self.trace.nodes[known_path[position]].sampled ): issuer = known_path[position] - self.tool_issuers.setdefault(alias, set()).add(issuer) calls[call.id] = (issuer, call.id) else: calls[call.id] = alias @@ -549,7 +547,7 @@ async def gate_tool_calls( self, node: int, dialect: DialectName | None = None, - native_tools: list[Tool] | None = None, + harness_history: Request | None = None, ) -> str | None: """Police the tool calls the assistant committed at `node` before the harness sees them: the request hooks run over the branch plus an empty result for each call, so a @@ -563,11 +561,20 @@ async def gate_tool_calls( self.tool_dialects[node] = dialect if not isinstance(assistant, AssistantMessage) or not assistant.tool_calls: return None - self.tool_catalogs[node] = graph.tool_catalog_hash(native_tools) known_path = graph.path_node_ids(self.trace, node) branch = [self.trace.nodes[i].message for i in known_path] tools = self.trace.nodes[known_path[0]].tools + if harness_history is None: + harness_history = Request(messages=branch[:-1], tools=tools) + self.tool_catalogs[node] = graph.tool_catalog_hash(harness_history.tools) + prefix = hashlib.blake2b(digest_size=16) + for message in [*harness_history.messages, assistant]: + prefix.update( + graph.replay_message_hash(self.trace, message, dialect).encode() + ) + prefix_hash = prefix.hexdigest() for call in assistant.tool_calls: + self.tool_issuers.setdefault((prefix_hash, call.id), set()).add(node) probe = ToolMessage(tool_call_id=call.id, content="", name=call.name) request = Request(messages=[*branch, probe], tools=tools or None) key = (node, call.id)