diff --git a/js/llm.ts b/js/llm.ts index 1e57232..59d6479 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -18,6 +18,10 @@ import { computeThreadTemplateVars, type ThreadTemplateVars, } from "./thread-utils"; +import { + computeTrajectoryTemplateVars, + templateUsesTrajectoryVariables, +} from "./trajectory-utils"; /** * Minimal interface for a Trace object that can provide thread data. @@ -25,6 +29,7 @@ import { */ export interface TraceForScorer { getThread(options?: { preprocessor?: string }): Promise; + getTrajectory?(): Promise; } // Thread-related template variable names that require preprocessor invocation @@ -342,6 +347,15 @@ export function LLMClassifierFromTemplate({ threadVars[name] = computed[name as keyof ThreadTemplateVars]; } } + if (runtimeArgs.trace && templateUsesTrajectoryVariables(promptTemplate)) { + if (!runtimeArgs.trace.getTrajectory) { + throw new Error("Trajectory variables need trace.getTrajectory()"); + } + Object.assign( + threadVars, + computeTrajectoryTemplateVars(await runtimeArgs.trace.getTrajectory()), + ); + } const prompt = promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX); diff --git a/js/trajectory-utils.ts b/js/trajectory-utils.ts new file mode 100644 index 0000000..f7ccb4b --- /dev/null +++ b/js/trajectory-utils.ts @@ -0,0 +1,88 @@ +/** + * Renders `trace.getTrajectory()` as plain text for LLM-as-a-judge templates. + * `{{trajectory}}` omits the agent's instructions; `{{trajectory_with_system}}` includes them. + */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +export function templateUsesTrajectoryVariables(template: string): boolean { + return /\{\{\s*trajectory(_with_system)?\s*\}\}/.test(template); +} + +export function computeTrajectoryTemplateVars(trajectories: any[]) { + return { + trajectory: render(trajectories, false), + trajectory_with_system: render(trajectories, true), + }; +} + +// One block per trajectory, separated by a blank line. +function render(trajectories: any[], withSystem: boolean): string { + return trajectories + .map((t) => renderTurns(t.turns ?? [], withSystem)) + .filter(Boolean) + .join("\n\n"); +} + +function renderTurns(turns: any[], withSystem: boolean): string { + const lines: string[] = []; + let lastSystem = ""; + for (const turn of turns) { + const request: any[] = turn.request ?? []; + const work: any[] = turn.work ?? []; + + // Some traces keep the instructions only in the LLM call. They can change between turns. + const llmMessages = work + .filter((s) => s.work?.type === "llm_analysis") + .flatMap((s) => s.work.work ?? []); + const systemTexts = [...request, ...llmMessages] + .filter((m) => m.role === "system") + .map((m) => text(m.content)); + const system = [...new Set(systemTexts)].join("\n"); + if (withSystem && system && system !== lastSystem) { + lines.push(`System: ${system}`); + lastSystem = system; + } + + for (const m of request) { + if (m.role === "user") lines.push(`User: ${text(m.content)}`); + } + + for (const step of work) { + // Skip steps that are not tool calls, such as span_type "task". + if (step.span_type !== "tool" || step.work?.type !== "tool_result") { + continue; + } + // When the tool call has an ID, Lingua moves the output into `content`. + const part = step.work.content?.find( + (p: any) => p.type === "tool_result", + ); + const output = part ? part.output : step.work.output; + const result = + step.error != null ? `error: ${json(step.error)}` : json(output); + const name = step.name || "tool"; + lines.push(`Tool: ${name}(${json(step.work.input)}) -> ${result}`); + } + + // An interrupted response is already cut off where it stopped. + let response = text(turn.response?.response); + if (turn.interrupted) response += " [interrupted]"; + if (response.trim()) lines.push(`Assistant: ${response.trim()}`); + } + return lines.join("\n"); +} + +// Message content is a string or a list of parts. Only text renders; audio and images don't. +function text(content: any): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter((p) => p?.type === "text") + .map((p) => p.text) + .join("\n"); +} + +// Tool inputs, outputs and errors: strings as they are, anything else as compact JSON. +function json(value: any): string { + if (value === undefined || value === null) return ""; + return typeof value === "string" ? value : JSON.stringify(value); +} diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 1a38712..570cc72 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -67,6 +67,7 @@ filter_system_messages_from_thread, template_uses_thread_variables, ) +from .trajectory_utils import compute_trajectory_template_vars, template_uses_trajectory_variables # Disable HTML escaping in chevron. chevron.renderer._html_escape = lambda x: x # type: ignore[attr-defined] @@ -378,6 +379,7 @@ def __init__( **extra_render_args, ): self._template_uses_thread_variables = template_uses_thread_variables(prompt_template) + self._template_uses_trajectory_variables = template_uses_trajectory_variables(prompt_template) choice_strings = list(choice_scores.keys()) # Use configured default model if not specified if model is None: @@ -458,18 +460,22 @@ def _request_args(self, output, expected, **kwargs): thread_vars: dict[str, object] = {} if trace is not None and self._template_uses_thread_variables: thread_vars = self._compute_thread_vars_sync(trace) + if trace is not None and self._template_uses_trajectory_variables: + thread_vars.update(compute_trajectory_template_vars(asyncio.run(trace.get_trajectory()))) # Thread vars come first so explicit render args can override. - return super()._request_args(output, expected, **thread_vars, **kwargs) + return super()._request_args(output, expected, **{**thread_vars, **kwargs}) async def _request_args_async(self, output, expected, **kwargs): trace = kwargs.get("trace") thread_vars: dict[str, object] = {} if trace is not None and self._template_uses_thread_variables: thread_vars = await self._compute_thread_vars_async(trace) + if trace is not None and self._template_uses_trajectory_variables: + thread_vars.update(compute_trajectory_template_vars(await trace.get_trajectory())) # Thread vars come first so explicit render args can override. - return super()._request_args(output, expected, **thread_vars, **kwargs) + return super()._request_args(output, expected, **{**thread_vars, **kwargs}) @classmethod def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = None, **kwargs): diff --git a/py/autoevals/trajectory_utils.py b/py/autoevals/trajectory_utils.py new file mode 100644 index 0000000..5e311e9 --- /dev/null +++ b/py/autoevals/trajectory_utils.py @@ -0,0 +1,82 @@ +"""Renders `trace.get_trajectory()` as plain text for LLM-as-a-judge templates. + +`{{trajectory}}` omits the agent's instructions; `{{trajectory_with_system}}` includes them. +""" + +import json +import re +from typing import Any + + +def template_uses_trajectory_variables(template: str) -> bool: + return re.search(r"\{\{\s*trajectory(_with_system)?\s*\}\}", template) is not None + + +def compute_trajectory_template_vars(trajectories: list[Any]) -> dict[str, str]: + return { + "trajectory": _render(trajectories, with_system=False), + "trajectory_with_system": _render(trajectories, with_system=True), + } + + +# One block per trajectory, separated by a blank line. +def _render(trajectories: list[Any], with_system: bool) -> str: + blocks = [_render_turns(t.get("turns") or [], with_system) for t in trajectories] + return "\n\n".join(b for b in blocks if b) + + +def _render_turns(turns: list[Any], with_system: bool) -> str: + lines: list[str] = [] + last_system = "" + for turn in turns: + request = turn.get("request") or [] + work = turn.get("work") or [] + + # Some traces keep the instructions only in the LLM call. They can change between turns. + llm_messages = [ + m for s in work if (s.get("work") or {}).get("type") == "llm_analysis" for m in s["work"].get("work") or [] + ] + system_texts = [_text(m.get("content")) for m in request + llm_messages if m.get("role") == "system"] + system = "\n".join(dict.fromkeys(system_texts)) + if with_system and system and system != last_system: + lines.append(f"System: {system}") + last_system = system + + lines += [f"User: {_text(m.get('content'))}" for m in request if m.get("role") == "user"] + + for step in work: + # Skip steps that are not tool calls, such as span_type "task". + step_work = step.get("work") or {} + if step.get("span_type") != "tool" or step_work.get("type") != "tool_result": + continue + # When the tool call has an ID, Lingua moves the output into `content`. + parts = [p for p in step_work.get("content") or [] if p.get("type") == "tool_result"] + output = parts[0].get("output") if parts else step_work.get("output") + error = step.get("error") + result = f"error: {_json(error)}" if error is not None else _json(output) + name = step.get("name") or "tool" + lines.append(f"Tool: {name}({_json(step_work.get('input'))}) -> {result}") + + # An interrupted response is already cut off where it stopped. + response = _text((turn.get("response") or {}).get("response")) + if turn.get("interrupted"): + response += " [interrupted]" + if response.strip(): + lines.append(f"Assistant: {response.strip()}") + return "\n".join(lines) + + +# Message content is a string or a list of parts. Only text renders; audio and images don't. +def _text(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + return "\n".join(p["text"] for p in content if isinstance(p, dict) and p.get("type") == "text") + + +# Tool inputs, outputs and errors: strings as they are, anything else as compact JSON. +def _json(value: Any) -> str: + if value is None: + return "" + return value if isinstance(value, str) else json.dumps(value, separators=(",", ":"), ensure_ascii=False)