Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/eval-set-format.md
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,12 @@ The `parts` array can contain text, function calls, or function responses. Most

Each `FunctionCall` has `name`, `args`, and `id`. Each `FunctionResponse` has `name`, `response`, and `id`. Match `id` values between calls and responses to pair them.

Trace conversion returns native ADK `InvocationEvents` in memory so tool calls
and results retain their source agent's name. EvalSet and `/api/convert` exports
keep the flat `IntermediateData` JSON above. Golden inputs accept either ADK
variant. Flat tool evidence with one captured agent uses that owner; multiple
captured agents require explicit event authors because flattening loses ownership.

## Which Evaluators Use Eval Sets

Not all evaluators require an eval set. Use `agentevals evaluator list --source builtin` to see which built-in evaluators do:
Expand Down
96 changes: 72 additions & 24 deletions src/agentevals/adk_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,15 @@
from collections.abc import Sequence
from typing import Any

from google.adk.evaluation.eval_case import EvalCase, IntermediateData, Invocation, InvocationEvent
from google.adk.evaluation.eval_case import (
EvalCase,
IntermediateData,
Invocation,
InvocationEvent,
InvocationEvents,
get_all_tool_calls,
get_all_tool_responses,
)
from google.adk.evaluation.eval_rubrics import Rubric, RubricContent
from google.adk.evaluation.eval_set import EvalSet
from google.genai import types as genai_types
Expand Down Expand Up @@ -44,11 +52,8 @@ def expected_from_case(case: EvalCase) -> ExpectedConversation:
turns = []
for inv in case.conversation or []:
data = inv.intermediate_data
calls: tuple[dict[str, Any], ...] = ()
responses: tuple[dict[str, Any], ...] = ()
if isinstance(data, IntermediateData):
calls = tuple({"id": c.id, "name": c.name, "arguments": c.args} for c in data.tool_uses)
responses = tuple({"id": r.id, "name": r.name, "response": r.response} for r in data.tool_responses)
calls = tuple({"id": c.id, "name": c.name, "arguments": c.args} for c in get_all_tool_calls(data))
responses = tuple({"id": r.id, "name": r.name, "response": r.response} for r in get_all_tool_responses(data))
turns.append(
ExpectedTurn(
user_text=_content_text(inv.user_content),
Expand Down Expand Up @@ -99,31 +104,55 @@ def _as_dict(value: Any, key: str) -> dict[str, Any]:
def to_adk_invocation(turn: Turn) -> Invocation:
user_text = text_of(turn.user_input)
final_text = text_of(turn.final_output)
tool_uses = [
genai_types.FunctionCall(id=t.call_id, name=t.name, args=_as_dict(t.arguments, "value"))
for t in turn.tool_calls
]
tool_responses = [
genai_types.FunctionResponse(id=t.call_id, name=t.name, response=_as_dict(t.result, "result"))
for t in turn.tool_calls
if t.result is not None
]
intermediate = [
(turn.agent_name or "agent", [genai_types.Part(text=text)])
events = []
for tool in turn.tool_calls:
author = tool.agent_name or turn.agent_name or "agent"
events.append(
InvocationEvent(
author=author,
content=genai_types.Content(
role="model",
parts=[
genai_types.Part(
function_call=genai_types.FunctionCall(
id=tool.call_id, name=tool.name, args=_as_dict(tool.arguments, "value")
)
)
],
),
)
)
if tool.result is not None:
events.append(
InvocationEvent(
author=author,
content=genai_types.Content(
role="user",
parts=[
genai_types.Part(
function_response=genai_types.FunctionResponse(
id=tool.call_id, name=tool.name, response=_as_dict(tool.result, "result")
)
)
],
),
)
)
events.extend(
InvocationEvent(
author=turn.agent_name or "agent",
content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)]),
)
for text in (text_of([m]) for m in turn.intermediate_outputs)
if text
]
)
return Invocation(
invocation_id=turn.ref.span_id,
user_content=genai_types.Content(role="user", parts=[genai_types.Part(text=user_text or "")]),
final_response=genai_types.Content(role="model", parts=[genai_types.Part(text=final_text)])
if final_text
else None,
intermediate_data=IntermediateData(
tool_uses=tool_uses,
tool_responses=tool_responses,
intermediate_responses=intermediate,
),
intermediate_data=InvocationEvents(invocation_events=events),
creation_timestamp=turn.start_ns / 1e9,
**_durations(turn),
)
Expand Down Expand Up @@ -216,7 +245,26 @@ def invocation_to_dict(inv: Invocation) -> dict[str, Any]:
if inv.final_response is not None:
out["final_response"] = inv.final_response.model_dump(mode="json", exclude_none=True)
if inv.intermediate_data is not None:
out["intermediate_data"] = inv.intermediate_data.model_dump(mode="json", exclude_none=True)
data = inv.intermediate_data
if isinstance(data, InvocationEvents):
# Export keeps the existing flat JSON; scoring retains the source event owners.
data = IntermediateData(
tool_uses=get_all_tool_calls(data),
tool_responses=get_all_tool_responses(data),
intermediate_responses=[
(event.author, parts)
for event in data.invocation_events
if event.content
and (
parts := [
part
for part in event.content.parts or []
if not part.function_call and not part.function_response
]
)
],
)
out["intermediate_data"] = data.model_dump(mode="json", exclude_none=True)
if inv.creation_timestamp is not None:
out["creation_timestamp"] = inv.creation_timestamp
return out
Expand Down
94 changes: 50 additions & 44 deletions src/agentevals/builtin_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,25 +122,37 @@ def _to_invocation_events(inv: Invocation, model_calls: Sequence[InvocationEvent
emits actual invocations as ``InvocationEvents``, and several evaluators read only that
variant of the ``IntermediateDataType`` union: ``hallucinations_v1`` and the multi-turn
Vertex metrics treat ``IntermediateData`` as an empty trajectory, so the judge never
sees tool outputs. agentevals' trace converters populate ``IntermediateData``, so every
built-in metric is fed through this adapter.
sees tool outputs. The bridge preserves source-owned events; older eval sets may still
carry flat ``IntermediateData`` and pass through this compatibility adapter.

Each tool call is followed by its response, matched by ``id`` first, then by tool name
in order among responses whose ``id`` matches no call (instrumentations disagree on call
ids, e.g. positional ids in LLM messages vs per-execution ids on tool spans). Responses
that still have no call are appended rather than dropped, because missing evidence makes
the hallucinations judge label grounded claims as unsupported. ADK authors calls and
responses with the agent name (no separate ``"tool"`` actor); ``"agent"`` mirrors that.
the hallucinations judge label grounded claims as unsupported. Legacy flat data uses
its sole captured agent owner, or ``"agent"`` when no ownership metadata is available.
Multiple captured owners are ambiguous after flattening and cannot be reconstructed.

*model_calls* are appended as given: content-less model call events that ADK's token and
inference call counts read, and that every other evaluator skips.
"""
from google.genai import types as genai_types

if inv.intermediate_data is None or isinstance(inv.intermediate_data, InvocationEvents):
return inv
if not model_calls:
return inv
events = list(inv.intermediate_data.invocation_events) if inv.intermediate_data else []
if events[-len(model_calls) :] != list(model_calls):
events.extend(model_calls)
return inv.model_copy(update={"intermediate_data": InvocationEvents(invocation_events=events)})

id_: IntermediateData = inv.intermediate_data
owners = list(inv.app_details.agent_details) if inv.app_details else []
if len(owners) > 1 and (id_.tool_uses or id_.tool_responses):
raise ValueError(
"Legacy flat tool evidence has multiple agent owners; source-owned invocation events are required"
)
owner = owners[0] if len(owners) == 1 else "agent"
call_ids = {tc.id for tc in id_.tool_uses if tc.id}
responses = list(id_.tool_responses)
consumed = [False] * len(responses)
Expand All @@ -158,15 +170,15 @@ def _claim(tool_call: genai_types.FunctionCall) -> genai_types.FunctionResponse

def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent:
return InvocationEvent(
author="agent",
author=owner,
content=genai_types.Content(role="user", parts=[genai_types.Part(function_response=tr)]),
)

events: list[InvocationEvent] = []
for tool_call in id_.tool_uses:
events.append(
InvocationEvent(
author="agent",
author=owner,
content=genai_types.Content(role="model", parts=[genai_types.Part(function_call=tool_call)]),
)
)
Expand All @@ -179,7 +191,7 @@ def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent:
for author, parts in id_.intermediate_responses:
events.append(
InvocationEvent(
author=author or "agent",
author=author or owner,
content=genai_types.Content(role="model", parts=list(parts)),
)
)
Expand All @@ -189,46 +201,40 @@ def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent:


def _enrich_app_details(invocations: list[Invocation]) -> list[Invocation]:
"""Synthesize minimal ``app_details`` so judges know which tools the agent had.

ADK judges read tool declarations from ``app_details.agent_details``. Without them,
``multi_turn_tool_use_quality_v1`` has no schema to compare calls against, and the
``hallucinations_v1`` / ``rubric_based_*`` prompts state "Agent has no tools." next to
the tool calls they show. Our trace converters do not populate ``app_details``, so we
synthesize a minimal record from tool names observed across the conversation.
"""
"""Fill missing event owners with names-only metadata, preserving captured declarations."""
from google.adk.evaluation.app_details import AgentDetails, AppDetails
from google.genai import types as genai_types

if any(inv.app_details and inv.app_details.agent_details for inv in invocations):
return invocations

tool_names: dict[str, None] = {}
enriched = []
for inv in invocations:
data = inv.intermediate_data
if data is None:
continue
if isinstance(data, IntermediateData):
for tc in data.tool_uses:
if tc.name:
tool_names.setdefault(tc.name)
elif isinstance(data, InvocationEvents):
for ev in data.invocation_events:
if not (ev.content and ev.content.parts):
continue
for part in ev.content.parts:
if part.function_call and part.function_call.name:
tool_names.setdefault(part.function_call.name)

if not tool_names:
return invocations

function_declarations = [genai_types.FunctionDeclaration(name=name) for name in tool_names]
tool = genai_types.Tool(function_declarations=function_declarations)
agent_details = AgentDetails(name="agent", instructions="", tool_declarations=[tool])
app_details = AppDetails(agent_details={"agent": agent_details})

return [inv.model_copy(update={"app_details": app_details}) for inv in invocations]
data = _to_invocation_events(inv).intermediate_data
captured = inv.app_details.agent_details if inv.app_details else {}
names_by_owner: dict[str, dict[str, None]] = {}
for event in data.invocation_events if data else []:
names = names_by_owner.setdefault(event.author, {})
for part in event.content.parts or [] if event.content else []:
if part.function_call and part.function_call.name:
names.setdefault(part.function_call.name)
missing = {
owner: AgentDetails(
name=owner,
tool_declarations=[
genai_types.Tool(
function_declarations=[genai_types.FunctionDeclaration(name=name) for name in names]
)
]
if names
else [],
)
for owner, names in names_by_owner.items()
if owner not in captured and (names or captured)
}
enriched.append(
inv.model_copy(update={"app_details": AppDetails(agent_details={**captured, **missing})})
if missing
else inv
)
return enriched


def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]:
Expand Down
15 changes: 6 additions & 9 deletions tests/test_builtin_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@
from agentevals.genai.extract import extract_conversation
from agentevals.loader import load_traces

SAMPLES_DIR = os.path.join(os.path.dirname(__file__), "..", "samples")
HELM_TRACE = os.path.join(SAMPLES_DIR, "helm.json")


def _invocation(
calls: list[tuple[str, str | None]],
Expand Down Expand Up @@ -257,17 +254,17 @@ async def test_evidence_kept_when_call_and_response_ids_disagree(self, fake_judg

assert result.score == 1.0

@pytest.mark.skipif(not os.path.exists(HELM_TRACE), reason="samples/helm.json not available")
async def test_replayed_trace_tool_output_reaches_judge(self, fake_judge):
judges = fake_judge(evidence="kagent-crds-0.7.14")
(inv,) = to_adk_invocations(extract_conversation(load_traces(HELM_TRACE)).turns)
assert isinstance(inv.intermediate_data, IntermediateData)
assert "kagent-crds-0.7.14" in str(inv.intermediate_data.tool_responses[0].response)
judges = fake_judge(evidence="temp_c")
fixture = os.path.join(os.path.dirname(__file__), "fixtures/otlp/adk_py/v2proj_span_only/S2.json")
(inv,) = to_adk_invocations(extract_conversation(load_traces(fixture)).turns)
assert isinstance(inv.intermediate_data, InvocationEvents)
assert get_all_tool_responses(inv.intermediate_data)[0].response["temp_c"] == 21

result = await evaluate_builtin_metric("hallucinations_v1", [inv], None, None, 0.5)

assert result.score == 1.0
assert "kagent-crds-0.7.14" in judges[0].validator_contexts[0]
assert "temp_c" in judges[0].validator_contexts[0]


class _RecordingEvaluator:
Expand Down
Loading
Loading