Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/agentevals/builtin_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,8 +296,12 @@ def attach_case_rubrics(
return attached, warnings


def has_rubric_scores(eval_result: EvaluationResult) -> bool:
return bool(eval_result.overall_rubric_scores) or any(r.rubric_scores for r in eval_result.per_invocation_results)


def extract_rubric_details(eval_result: EvaluationResult) -> dict[str, Any]:
"""Per-rubric scores, per invocation and overall, from a rubric metric."""
"""Per-rubric scores and rationale, per invocation and overall, from any metric that returns them."""

def _score(rubric_score: Any) -> dict[str, Any]:
return {
Expand Down Expand Up @@ -406,6 +410,10 @@ def get_evaluator(eval_metric: EvalMetric) -> Evaluator:
"google.adk.evaluation.trajectory_evaluator",
"TrajectoryEvaluator",
),
"final_response_match_v2": (
"agentevals.judges",
"FinalResponseMatchV2WithReasoning",
),
}

if name in _DIRECT_EVALUATORS:
Expand Down Expand Up @@ -609,7 +617,7 @@ async def evaluate_builtin_metric(
details = None
if metric_name == "tool_trajectory_avg_score":
details = extract_trajectory_details(eval_result)
elif metric_name in RUBRIC_METRIC_TYPES:
elif metric_name in RUBRIC_METRIC_TYPES or has_rubric_scores(eval_result):
details = extract_rubric_details(eval_result)
if rubric_warnings:
details["warnings"] = rubric_warnings
Expand Down
66 changes: 66 additions & 0 deletions src/agentevals/judges.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""ADK judge evaluators that keep the judge's reasoning next to its score."""

from __future__ import annotations

import json
import re

from google.adk.evaluation.eval_rubrics import RubricScore
from google.adk.evaluation.final_response_match_v2 import FinalResponseMatchV2Evaluator
from google.adk.evaluation.llm_as_judge import AutoRaterScore
from google.adk.evaluation.llm_as_judge_utils import get_text_from_content
from google.adk.models.llm_response import LlmResponse

FINAL_RESPONSE_MATCH_RUBRIC_ID = "final_response_match"

_REASONING_KEY = re.compile(r'"reasoning"\s*:\s*')


def parse_judge_reasoning(response_text: str) -> str | None:
"""Return the ``reasoning`` value from a judge reply, or None.

ADK's prompt asks for JSON with a trailing comma and an unquoted label placeholder, so judges
often reply with text ``json.loads`` rejects. Decoding only the value after the key tolerates
that, along with code fences and prose around the object.
"""
match = _REASONING_KEY.search(response_text)
if match is None:
return None
try:
value, _ = json.JSONDecoder().raw_decode(response_text, match.end())
except json.JSONDecodeError:
return None
if isinstance(value, list):
value = "\n".join(item for item in value if isinstance(item, str))
if not isinstance(value, str):
return None
return value.strip() or None


class FinalResponseMatchV2WithReasoning(FinalResponseMatchV2Evaluator):
"""``final_response_match_v2`` that also returns the judge's reasoning as a rubric score.

The score comes from ADK's own label parsing and is never changed by the reasoning. The
rationale rides on ``rubric_scores`` of each sample, and the majority vote returns a whole
winning sample, so the reasoning reported is the one behind the reported verdict.

Remove once ADK returns the judge reasoning for this metric itself.
"""

def convert_auto_rater_response_to_score(self, llm_response: LlmResponse) -> AutoRaterScore:
auto_rater_score = super().convert_auto_rater_response_to_score(llm_response)
if auto_rater_score.score is None:
return auto_rater_score
reasoning = parse_judge_reasoning(get_text_from_content(llm_response.content) or "")
if reasoning is None:
return auto_rater_score
return AutoRaterScore(
score=auto_rater_score.score,
rubric_scores=[
RubricScore(
rubric_id=FINAL_RESPONSE_MATCH_RUBRIC_ID,
rationale=reasoning,
score=auto_rater_score.score,
)
],
)
Loading