From 35bbe136a3132db293c91472231d770a4cfab5e3 Mon Sep 17 00:00:00 2001 From: erauner Date: Sun, 1 Nov 2026 17:20:22 -0600 Subject: [PATCH 1/2] fix(adk-bridge): preserve tool event agent identities --- docs/eval-set-format.md | 6 + src/agentevals/adk_bridge.py | 96 ++++++++++++---- src/agentevals/builtin_metrics.py | 94 ++++++++-------- tests/test_builtin_metrics.py | 4 +- tests/test_invocation_owners.py | 178 ++++++++++++++++++++++++++++++ 5 files changed, 308 insertions(+), 70 deletions(-) create mode 100644 tests/test_invocation_owners.py diff --git a/docs/eval-set-format.md b/docs/eval-set-format.md index 96cd1d1..9a32f71 100644 --- a/docs/eval-set-format.md +++ b/docs/eval-set-format.md @@ -250,6 +250,12 @@ The `parts` array can contain text, function calls, or function responses. Most Each `FunctionCall` has `name`, `args`, and `id`. Each `FunctionResponse` has `name`, `response`, and `id`. Match `id` values between calls and responses to pair them. +Trace conversion returns native ADK `InvocationEvents` in memory so tool calls +and results retain their source agent's name. EvalSet and `/api/convert` exports +keep the flat `IntermediateData` JSON above. Golden inputs accept either ADK +variant. Flat tool evidence with one captured agent uses that owner; multiple +captured agents require explicit event authors because flattening loses ownership. + ## Which Evaluators Use Eval Sets Not all evaluators require an eval set. Use `agentevals evaluator list --source builtin` to see which built-in evaluators do: diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index 1a67d7c..e9bfed6 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -13,7 +13,15 @@ from collections.abc import Sequence from typing import Any -from google.adk.evaluation.eval_case import EvalCase, IntermediateData, Invocation, InvocationEvent +from google.adk.evaluation.eval_case import ( + EvalCase, + IntermediateData, + Invocation, + InvocationEvent, + InvocationEvents, + get_all_tool_calls, + get_all_tool_responses, +) from google.adk.evaluation.eval_rubrics import Rubric, RubricContent from google.adk.evaluation.eval_set import EvalSet from google.genai import types as genai_types @@ -44,11 +52,8 @@ def expected_from_case(case: EvalCase) -> ExpectedConversation: turns = [] for inv in case.conversation or []: data = inv.intermediate_data - calls: tuple[dict[str, Any], ...] = () - responses: tuple[dict[str, Any], ...] = () - if isinstance(data, IntermediateData): - calls = tuple({"id": c.id, "name": c.name, "arguments": c.args} for c in data.tool_uses) - responses = tuple({"id": r.id, "name": r.name, "response": r.response} for r in data.tool_responses) + calls = tuple({"id": c.id, "name": c.name, "arguments": c.args} for c in get_all_tool_calls(data)) + responses = tuple({"id": r.id, "name": r.name, "response": r.response} for r in get_all_tool_responses(data)) turns.append( ExpectedTurn( user_text=_content_text(inv.user_content), @@ -99,31 +104,55 @@ def _as_dict(value: Any, key: str) -> dict[str, Any]: def to_adk_invocation(turn: Turn) -> Invocation: user_text = text_of(turn.user_input) final_text = text_of(turn.final_output) - tool_uses = [ - genai_types.FunctionCall(id=t.call_id, name=t.name, args=_as_dict(t.arguments, "value")) - for t in turn.tool_calls - ] - tool_responses = [ - genai_types.FunctionResponse(id=t.call_id, name=t.name, response=_as_dict(t.result, "result")) - for t in turn.tool_calls - if t.result is not None - ] - intermediate = [ - (turn.agent_name or "agent", [genai_types.Part(text=text)]) + events = [] + for tool in turn.tool_calls: + author = tool.agent_name or turn.agent_name or "agent" + events.append( + InvocationEvent( + author=author, + content=genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=genai_types.FunctionCall( + id=tool.call_id, name=tool.name, args=_as_dict(tool.arguments, "value") + ) + ) + ], + ), + ) + ) + if tool.result is not None: + events.append( + InvocationEvent( + author=author, + content=genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=genai_types.FunctionResponse( + id=tool.call_id, name=tool.name, response=_as_dict(tool.result, "result") + ) + ) + ], + ), + ) + ) + events.extend( + InvocationEvent( + author=turn.agent_name or "agent", + content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)]), + ) for text in (text_of([m]) for m in turn.intermediate_outputs) if text - ] + ) return Invocation( invocation_id=turn.ref.span_id, user_content=genai_types.Content(role="user", parts=[genai_types.Part(text=user_text or "")]), final_response=genai_types.Content(role="model", parts=[genai_types.Part(text=final_text)]) if final_text else None, - intermediate_data=IntermediateData( - tool_uses=tool_uses, - tool_responses=tool_responses, - intermediate_responses=intermediate, - ), + intermediate_data=InvocationEvents(invocation_events=events), creation_timestamp=turn.start_ns / 1e9, **_durations(turn), ) @@ -216,7 +245,26 @@ def invocation_to_dict(inv: Invocation) -> dict[str, Any]: if inv.final_response is not None: out["final_response"] = inv.final_response.model_dump(mode="json", exclude_none=True) if inv.intermediate_data is not None: - out["intermediate_data"] = inv.intermediate_data.model_dump(mode="json", exclude_none=True) + data = inv.intermediate_data + if isinstance(data, InvocationEvents): + # Export keeps the existing flat JSON; scoring retains the source event owners. + data = IntermediateData( + tool_uses=get_all_tool_calls(data), + tool_responses=get_all_tool_responses(data), + intermediate_responses=[ + (event.author, parts) + for event in data.invocation_events + if event.content + and ( + parts := [ + part + for part in event.content.parts or [] + if not part.function_call and not part.function_response + ] + ) + ], + ) + out["intermediate_data"] = data.model_dump(mode="json", exclude_none=True) if inv.creation_timestamp is not None: out["creation_timestamp"] = inv.creation_timestamp return out diff --git a/src/agentevals/builtin_metrics.py b/src/agentevals/builtin_metrics.py index 64e082d..5007e80 100644 --- a/src/agentevals/builtin_metrics.py +++ b/src/agentevals/builtin_metrics.py @@ -122,15 +122,16 @@ def _to_invocation_events(inv: Invocation, model_calls: Sequence[InvocationEvent emits actual invocations as ``InvocationEvents``, and several evaluators read only that variant of the ``IntermediateDataType`` union: ``hallucinations_v1`` and the multi-turn Vertex metrics treat ``IntermediateData`` as an empty trajectory, so the judge never - sees tool outputs. agentevals' trace converters populate ``IntermediateData``, so every - built-in metric is fed through this adapter. + sees tool outputs. The bridge preserves source-owned events; older eval sets may still + carry flat ``IntermediateData`` and pass through this compatibility adapter. Each tool call is followed by its response, matched by ``id`` first, then by tool name in order among responses whose ``id`` matches no call (instrumentations disagree on call ids, e.g. positional ids in LLM messages vs per-execution ids on tool spans). Responses that still have no call are appended rather than dropped, because missing evidence makes - the hallucinations judge label grounded claims as unsupported. ADK authors calls and - responses with the agent name (no separate ``"tool"`` actor); ``"agent"`` mirrors that. + the hallucinations judge label grounded claims as unsupported. Legacy flat data uses + its sole captured agent owner, or ``"agent"`` when no ownership metadata is available. + Multiple captured owners are ambiguous after flattening and cannot be reconstructed. *model_calls* are appended as given: content-less model call events that ADK's token and inference call counts read, and that every other evaluator skips. @@ -138,9 +139,20 @@ def _to_invocation_events(inv: Invocation, model_calls: Sequence[InvocationEvent from google.genai import types as genai_types if inv.intermediate_data is None or isinstance(inv.intermediate_data, InvocationEvents): - return inv + if not model_calls: + return inv + events = list(inv.intermediate_data.invocation_events) if inv.intermediate_data else [] + if events[-len(model_calls) :] != list(model_calls): + events.extend(model_calls) + return inv.model_copy(update={"intermediate_data": InvocationEvents(invocation_events=events)}) id_: IntermediateData = inv.intermediate_data + owners = list(inv.app_details.agent_details) if inv.app_details else [] + if len(owners) > 1 and (id_.tool_uses or id_.tool_responses): + raise ValueError( + "Legacy flat tool evidence has multiple agent owners; source-owned invocation events are required" + ) + owner = owners[0] if len(owners) == 1 else "agent" call_ids = {tc.id for tc in id_.tool_uses if tc.id} responses = list(id_.tool_responses) consumed = [False] * len(responses) @@ -158,7 +170,7 @@ def _claim(tool_call: genai_types.FunctionCall) -> genai_types.FunctionResponse def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent: return InvocationEvent( - author="agent", + author=owner, content=genai_types.Content(role="user", parts=[genai_types.Part(function_response=tr)]), ) @@ -166,7 +178,7 @@ def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent: for tool_call in id_.tool_uses: events.append( InvocationEvent( - author="agent", + author=owner, content=genai_types.Content(role="model", parts=[genai_types.Part(function_call=tool_call)]), ) ) @@ -179,7 +191,7 @@ def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent: for author, parts in id_.intermediate_responses: events.append( InvocationEvent( - author=author or "agent", + author=author or owner, content=genai_types.Content(role="model", parts=list(parts)), ) ) @@ -189,46 +201,40 @@ def _response_event(tr: genai_types.FunctionResponse) -> InvocationEvent: def _enrich_app_details(invocations: list[Invocation]) -> list[Invocation]: - """Synthesize minimal ``app_details`` so judges know which tools the agent had. - - ADK judges read tool declarations from ``app_details.agent_details``. Without them, - ``multi_turn_tool_use_quality_v1`` has no schema to compare calls against, and the - ``hallucinations_v1`` / ``rubric_based_*`` prompts state "Agent has no tools." next to - the tool calls they show. Our trace converters do not populate ``app_details``, so we - synthesize a minimal record from tool names observed across the conversation. - """ + """Fill missing event owners with names-only metadata, preserving captured declarations.""" from google.adk.evaluation.app_details import AgentDetails, AppDetails from google.genai import types as genai_types - if any(inv.app_details and inv.app_details.agent_details for inv in invocations): - return invocations - - tool_names: dict[str, None] = {} + enriched = [] for inv in invocations: - data = inv.intermediate_data - if data is None: - continue - if isinstance(data, IntermediateData): - for tc in data.tool_uses: - if tc.name: - tool_names.setdefault(tc.name) - elif isinstance(data, InvocationEvents): - for ev in data.invocation_events: - if not (ev.content and ev.content.parts): - continue - for part in ev.content.parts: - if part.function_call and part.function_call.name: - tool_names.setdefault(part.function_call.name) - - if not tool_names: - return invocations - - function_declarations = [genai_types.FunctionDeclaration(name=name) for name in tool_names] - tool = genai_types.Tool(function_declarations=function_declarations) - agent_details = AgentDetails(name="agent", instructions="", tool_declarations=[tool]) - app_details = AppDetails(agent_details={"agent": agent_details}) - - return [inv.model_copy(update={"app_details": app_details}) for inv in invocations] + data = _to_invocation_events(inv).intermediate_data + captured = inv.app_details.agent_details if inv.app_details else {} + names_by_owner: dict[str, dict[str, None]] = {} + for event in data.invocation_events if data else []: + names = names_by_owner.setdefault(event.author, {}) + for part in event.content.parts or [] if event.content else []: + if part.function_call and part.function_call.name: + names.setdefault(part.function_call.name) + missing = { + owner: AgentDetails( + name=owner, + tool_declarations=[ + genai_types.Tool( + function_declarations=[genai_types.FunctionDeclaration(name=name) for name in names] + ) + ] + if names + else [], + ) + for owner, names in names_by_owner.items() + if owner not in captured and (names or captured) + } + enriched.append( + inv.model_copy(update={"app_details": AppDetails(agent_details={**captured, **missing})}) + if missing + else inv + ) + return enriched def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]: diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index b274912..ac552e8 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -261,8 +261,8 @@ async def test_evidence_kept_when_call_and_response_ids_disagree(self, fake_judg async def test_replayed_trace_tool_output_reaches_judge(self, fake_judge): judges = fake_judge(evidence="kagent-crds-0.7.14") (inv,) = to_adk_invocations(extract_conversation(load_traces(HELM_TRACE)).turns) - assert isinstance(inv.intermediate_data, IntermediateData) - assert "kagent-crds-0.7.14" in str(inv.intermediate_data.tool_responses[0].response) + assert isinstance(inv.intermediate_data, InvocationEvents) + assert "kagent-crds-0.7.14" in str(get_all_tool_responses(inv.intermediate_data)[0].response) result = await evaluate_builtin_metric("hallucinations_v1", [inv], None, None, 0.5) diff --git a/tests/test_invocation_owners.py b/tests/test_invocation_owners.py new file mode 100644 index 0000000..81042db --- /dev/null +++ b/tests/test_invocation_owners.py @@ -0,0 +1,178 @@ +"""Native event ownership survives conversion without changing exported JSON.""" + +import json + +import pytest +from google.adk.evaluation.app_details import AgentDetails, AppDetails +from google.adk.evaluation.eval_case import EvalCase, IntermediateData, InvocationEvents +from google.genai import types + +from agentevals.adk_bridge import expected_from_case, invocation_to_dict, to_adk_invocation +from agentevals.builtin_metrics import _enrich_app_details, _to_invocation_events, build_eval_metric, get_evaluator +from agentevals.genai.model import ToolCall, Turn +from agentevals.otel.model import SpanRef + + +def source_turn(): + return Turn( + ref=SpanRef("synthetic-trace", "turn"), + agent_name="routing-agent", + user_input=[{"parts": [{"type": "text", "content": "Which field is missing?"}]}], + final_output=[{"parts": [{"type": "text", "content": "Which severity?"}]}], + intermediate_outputs=[{"parts": [{"type": "text", "content": "Checking the request."}]}], + tool_calls=[ + ToolCall(agent_name="routing_agent", name="lookup", call_id="one", arguments={"id": 1}, result={"v": 1}), + ToolCall(agent_name="specialist", name="lookup", call_id="two", arguments=[2], result=[3]), + ToolCall(agent_name="specialist", name="pending", call_id="three"), + ], + ) + + +def test_source_tool_owners_are_preserved_before_flattening(): + turn = source_turn() + before = turn.model_dump(mode="json") + invocation = to_adk_invocation(turn) + assert isinstance(invocation.intermediate_data, InvocationEvents) + events = invocation.intermediate_data.invocation_events + assert [event.author for event in events] == [ + "routing_agent", + "routing_agent", + "specialist", + "specialist", + "specialist", + "routing-agent", + ] + assert [event.content.role for event in events] == ["model", "user", "model", "user", "model", "model"] + assert events[0].content.parts[0].function_call.id == events[1].content.parts[0].function_response.id == "one" + assert events[2].content.parts[0].function_call.id == events[3].content.parts[0].function_response.id == "two" + assert turn.model_dump(mode="json") == before + + +@pytest.mark.parametrize( + "tool_owner,turn_owner,expected", [("worker", "root", "worker"), (None, "root", "root"), (None, None, "agent")] +) +def test_missing_tool_owner_uses_turn_then_agent(tool_owner, turn_owner, expected): + turn = Turn( + ref=SpanRef("trace", "turn"), + agent_name=turn_owner, + tool_calls=[ToolCall(agent_name=tool_owner, name="lookup", result={})], + ) + assert [event.author for event in to_adk_invocation(turn).intermediate_data.invocation_events] == [ + expected, + expected, + ] + + +def test_export_keeps_existing_flat_json_exactly(): + invocation = to_adk_invocation(source_turn()) + expected = { + "invocation_id": "turn", + "user_content": {"parts": [{"text": "Which field is missing?"}], "role": "user"}, + "final_response": {"parts": [{"text": "Which severity?"}], "role": "model"}, + "intermediate_data": { + "tool_uses": [ + {"id": "one", "args": {"id": 1}, "name": "lookup"}, + {"id": "two", "args": {"value": [2]}, "name": "lookup"}, + {"id": "three", "args": {}, "name": "pending"}, + ], + "tool_responses": [ + {"id": "one", "name": "lookup", "response": {"v": 1}}, + {"id": "two", "name": "lookup", "response": {"result": [3]}}, + ], + "intermediate_responses": [["routing-agent", [{"text": "Checking the request."}]]], + }, + "creation_timestamp": 0.0, + } + assert json.dumps(invocation_to_dict(invocation)) == json.dumps(expected) + + +@pytest.mark.parametrize("native", [False, True]) +def test_expected_tools_accept_both_native_variants(native): + invocation = to_adk_invocation(source_turn()) + if not native: + invocation = invocation.model_validate(invocation_to_dict(invocation)) + expected = expected_from_case(EvalCase(eval_id="synthetic", conversation=[invocation])).turns[0] + assert [call["id"] for call in expected.tool_calls] == ["one", "two", "three"] + assert [response["id"] for response in expected.tool_responses] == ["one", "two"] + + +def test_named_agent_reaches_native_final_prompt_with_captured_metadata(): + turn = source_turn() + turn.tool_calls = turn.tool_calls[:1] + turn.intermediate_outputs = [] + invocation = to_adk_invocation(turn) + invocation.app_details = AppDetails(agent_details={"routing_agent": AgentDetails(name="routing_agent")}) + prepared = _to_invocation_events(invocation) + metric = build_eval_metric( + "rubric_based_final_response_quality_v1", None, 1.0, rubrics=["Asks for the missing field."] + ) + prompt = get_evaluator(metric).format_auto_rater_prompt(prepared, None) + assert "Which severity?" in prompt and "lookup" in prompt + + +def test_legacy_single_owner_is_used_without_overwriting_native_authors(): + invocation = to_adk_invocation(source_turn()) + invocation = invocation.model_validate(invocation_to_dict(invocation)) + assert isinstance(invocation.intermediate_data, IntermediateData) + invocation.app_details = AppDetails(agent_details={"named": AgentDetails(name="named")}) + events = _to_invocation_events(invocation).intermediate_data.invocation_events + assert all(event.author == "named" for event in events[:-1]) + assert events[-1].author == "routing-agent" + native = to_adk_invocation(source_turn()) + native.app_details = invocation.app_details + assert _to_invocation_events(native) is native + + +def test_legacy_multiple_owners_fail_before_native_judging(): + invocation = to_adk_invocation(source_turn()) + invocation = invocation.model_validate(invocation_to_dict(invocation)) + invocation.app_details = AppDetails( + agent_details={name: AgentDetails(name=name) for name in ("agent", "specialist")} + ) + before = invocation.model_dump(mode="json") + with pytest.raises(ValueError, match="multiple agent owners"): + _to_invocation_events(invocation) + assert invocation.model_dump(mode="json") == before + + +def test_names_only_fallback_groups_owners_and_preserves_captured_declarations(): + invocation = to_adk_invocation(source_turn()) + declaration = types.Tool( + function_declarations=[types.FunctionDeclaration(name="uncalled", description="Captured.")] + ) + invocation.app_details = AppDetails( + agent_details={"routing_agent": AgentDetails(name="routing_agent", tool_declarations=[declaration])} + ) + before = invocation.model_dump(mode="json") + (enriched,) = _enrich_app_details([invocation]) + agents = enriched.app_details.agent_details + assert agents["routing_agent"].tool_declarations == [declaration] + assert [item.name for item in agents["specialist"].tool_declarations[0].function_declarations] == [ + "lookup", + "pending", + ] + assert invocation.model_dump(mode="json") == before + + +def test_native_accounting_is_appended_once_without_changing_tool_events(): + from agentevals.adk_bridge import model_call_events + from agentevals.genai.model import LlmCall, Usage + + turn = source_turn() + turn.llm_calls = [ + LlmCall( + ref=turn.ref, + operation="chat", + agent_name="routing_agent", + request_model="synthetic", + usage=Usage(input_tokens=5), + ) + ] + invocation = to_adk_invocation(turn) + original = invocation.model_dump(mode="json") + accounting = model_call_events(turn) + adapted = _to_invocation_events(invocation, accounting) + assert adapted.intermediate_data.invocation_events[:-1] == invocation.intermediate_data.invocation_events + assert adapted.intermediate_data.invocation_events[-1] == accounting[0] + assert _to_invocation_events(adapted, accounting) == adapted + assert invocation.model_dump(mode="json") == original From 67f52af9f201e9c95b6d6e8f2d8917b233a01b60 Mon Sep 17 00:00:00 2001 From: erauner Date: Mon, 2 Nov 2026 15:26:33 -0600 Subject: [PATCH 2/2] test(adk-bridge): evaluate semconv identities through native judge --- tests/test_builtin_metrics.py | 13 ++++----- tests/test_invocation_owners.py | 52 ++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index ac552e8..b7cd0cb 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -33,9 +33,6 @@ from agentevals.genai.extract import extract_conversation from agentevals.loader import load_traces -SAMPLES_DIR = os.path.join(os.path.dirname(__file__), "..", "samples") -HELM_TRACE = os.path.join(SAMPLES_DIR, "helm.json") - def _invocation( calls: list[tuple[str, str | None]], @@ -257,17 +254,17 @@ async def test_evidence_kept_when_call_and_response_ids_disagree(self, fake_judg assert result.score == 1.0 - @pytest.mark.skipif(not os.path.exists(HELM_TRACE), reason="samples/helm.json not available") async def test_replayed_trace_tool_output_reaches_judge(self, fake_judge): - judges = fake_judge(evidence="kagent-crds-0.7.14") - (inv,) = to_adk_invocations(extract_conversation(load_traces(HELM_TRACE)).turns) + judges = fake_judge(evidence="temp_c") + fixture = os.path.join(os.path.dirname(__file__), "fixtures/otlp/adk_py/v2proj_span_only/S2.json") + (inv,) = to_adk_invocations(extract_conversation(load_traces(fixture)).turns) assert isinstance(inv.intermediate_data, InvocationEvents) - assert "kagent-crds-0.7.14" in str(get_all_tool_responses(inv.intermediate_data)[0].response) + assert get_all_tool_responses(inv.intermediate_data)[0].response["temp_c"] == 21 result = await evaluate_builtin_metric("hallucinations_v1", [inv], None, None, 0.5) assert result.score == 1.0 - assert "kagent-crds-0.7.14" in judges[0].validator_contexts[0] + assert "temp_c" in judges[0].validator_contexts[0] class _RecordingEvaluator: diff --git a/tests/test_invocation_owners.py b/tests/test_invocation_owners.py index 81042db..dccce58 100644 --- a/tests/test_invocation_owners.py +++ b/tests/test_invocation_owners.py @@ -1,15 +1,26 @@ """Native event ownership survives conversion without changing exported JSON.""" import json +from pathlib import Path import pytest from google.adk.evaluation.app_details import AgentDetails, AppDetails from google.adk.evaluation.eval_case import EvalCase, IntermediateData, InvocationEvents +from google.adk.models.base_llm import BaseLlm +from google.adk.models.llm_response import LlmResponse from google.genai import types from agentevals.adk_bridge import expected_from_case, invocation_to_dict, to_adk_invocation -from agentevals.builtin_metrics import _enrich_app_details, _to_invocation_events, build_eval_metric, get_evaluator +from agentevals.builtin_metrics import ( + _enrich_app_details, + _to_invocation_events, + build_eval_metric, + evaluate_builtin_metric, + get_evaluator, +) +from agentevals.genai.extract import extract_conversation from agentevals.genai.model import ToolCall, Turn +from agentevals.loader import load_traces from agentevals.otel.model import SpanRef @@ -110,6 +121,45 @@ def test_named_agent_reaches_native_final_prompt_with_captured_metadata(): assert "Which severity?" in prompt and "lookup" in prompt +@pytest.mark.parametrize("captured_metadata", [False, True]) +async def test_semconv_trace_completes_native_final_response_evaluation(monkeypatch, captured_metadata): + from agentevals import builtin_metrics + + fixture = Path(__file__).parent / "fixtures/otlp/adk_py/v2proj_span_only/S2.json" + (turn,) = extract_conversation(load_traces(str(fixture))).turns + invocation = to_adk_invocation(turn) + if captured_metadata: + invocation.app_details = AppDetails(agent_details={"weather_agent": AgentDetails(name="weather_agent")}) + prompts = [] + + class Judge(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + prompts.append(llm_request.contents[0].parts[0].text) + yield LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + text="ID: rubric_0\nProperty: Uses captured evidence.\nRationale: fake\nVerdict: yes" + ) + ], + ) + ) + + metric_name = "rubric_based_final_response_quality_v1" + evaluator = get_evaluator(build_eval_metric(metric_name, None, 1.0, rubrics=["Uses captured evidence."])) + evaluator._judge_model = Judge(model="fake-judge") + monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda _: evaluator) + + result = await evaluate_builtin_metric( + metric_name, [invocation], None, None, 1.0, rubrics=["Uses captured evidence."] + ) + + assert result.error is None + assert result.score == 1.0 + assert prompts and all("weather_agent" in prompt and "get_weather" in prompt for prompt in prompts) + + def test_legacy_single_owner_is_used_without_overwriting_native_authors(): invocation = to_adk_invocation(source_turn()) invocation = invocation.model_validate(invocation_to_dict(invocation))