Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions verifiers/v1/dialects/anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import re
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from itertools import accumulate

from anthropic.types import Message as AnthropicMessage
from anthropic.types import Usage as AnthropicUsage
Expand Down Expand Up @@ -241,6 +242,10 @@ def content_to_wire(content) -> str | list[dict]:
return blocks


def _text_block_offsets(blocks: list[str]) -> list[int] | None:
return list(accumulate(map(len, blocks[:-1]))) or None


def parse_messages(body: dict) -> Messages:
"""The request's top-level `system` + `messages` -> typed messages. Assistant turns fold
their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's
Expand All @@ -260,7 +265,7 @@ def parse_messages(body: dict) -> Messages:
block for block in blocks if block["type"] not in ("text", "tool_use")
]
state.sort(key=lambda block: THINKING_ORDER.get(block["type"], 2))
text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text")
text = [b.get("text", "") for b in blocks if b.get("type") == "text"]
reasoning = "".join(
b.get("thinking", "") for b in blocks if b.get("type") == "thinking"
)
Expand All @@ -276,7 +281,8 @@ def parse_messages(body: dict) -> Messages:
]
prompt.append(
AssistantMessage(
content=text or None,
content="".join(text) or None,
text_block_offsets=_text_block_offsets(text),
reasoning_content=reasoning or None,
tool_calls=calls or None,
provider_state=state or None,
Expand Down Expand Up @@ -351,6 +357,7 @@ def response_from_wire(message: AnthropicMessage) -> Response:
model=message.model,
message=AssistantMessage(
content="".join(content) or None,
text_block_offsets=_text_block_offsets(content),
reasoning_content="".join(reasoning) or None,
tool_calls=calls or None,
provider_state=state or None,
Expand Down
7 changes: 3 additions & 4 deletions verifiers/v1/harnesses/claude_code/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from verifiers.v1.configs.harness import DialectName, HarnessConfig, PinnedVersion
from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
from verifiers.v1.harnesses.utils.replay import drop_blank_text
from verifiers.v1.harnesses.utils.replay import drop_blank_text_blocks
from verifiers.v1.runtimes import Runtime
from verifiers.v1.task import TaskData
from verifiers.v1.trace import Trace
Expand Down Expand Up @@ -44,10 +44,9 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
SUPPORTS_SKILLS = True
SUPPORTS_TOOL_INTERCEPTION = True

# Claude Code drops whitespace-only text blocks on replay. Closed source: observed
# with 2.1.314, recheck on version bumps.
# Claude Code 2.1.327 filters each whitespace-only text block on replay.
def replay_view(self, message: Message, dialect: DialectName | None) -> Message:
return drop_blank_text(message)
return drop_blank_text_blocks(message)

async def setup(self, runtime: Runtime) -> None:
await ensure_node(runtime)
Expand Down
19 changes: 19 additions & 0 deletions verifiers/v1/harnesses/utils/replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
`Harness.replay_view` overrides. Each function is idempotent and only normalizes
whitespace, so the replayed and the sampled form of a turn map to the same view."""

from itertools import pairwise

from verifiers.v1.types import AssistantMessage, Message


Expand All @@ -17,6 +19,23 @@ def drop_blank_text(message: Message) -> Message:
return message


def drop_blank_text_blocks(message: Message) -> Message:
"""Drop independently blank blocks while preserving whitespace inside other blocks."""
if not isinstance(message, AssistantMessage) or not message.text_block_offsets:
return drop_blank_text(message)
text = message.content or ""
offsets = (0, *message.text_block_offsets, len(text))
if any(start > end for start, end in pairwise(offsets)):
return drop_blank_text(message)
blocks = (text[start:end] for start, end in pairwise(offsets))
return message.model_copy(
update={
"content": "".join(block for block in blocks if block.strip()) or None,
"text_block_offsets": None,
}
)


def strip_text(message: Message) -> Message:
"""Clients that store assistant text stripped, and pad a missing reasoning field
(or Anthropic thinking block) with a single space on replay."""
Expand Down
4 changes: 4 additions & 0 deletions verifiers/v1/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ class ToolCall(BaseModel):
class AssistantMessage(BaseModel):
role: Literal["assistant"] = "assistant"
content: str | None = None
text_block_offsets: list[int] | None = Field(
default=None, exclude_if=lambda value: value is None
)
"""Character offsets between native text blocks concatenated into content."""
reasoning_content: str | None = None
tool_calls: list[ToolCall] | None = None
provider_state: list[dict[str, Any]] | None = None
Expand Down
Loading