Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions js/llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,18 @@ import {
computeThreadTemplateVars,
type ThreadTemplateVars,
} from "./thread-utils";
import {
computeTrajectoryTemplateVars,
templateUsesTrajectoryVariables,
} from "./trajectory-utils";

/**
* Minimal interface for a Trace object that can provide thread data.
* This is compatible with the Trace interface from the braintrust SDK.
*/
export interface TraceForScorer {
getThread(options?: { preprocessor?: string }): Promise<unknown[]>;
getTrajectory?(): Promise<unknown[]>;
}

// Thread-related template variable names that require preprocessor invocation
Expand Down Expand Up @@ -342,6 +347,15 @@ export function LLMClassifierFromTemplate<RenderArgs>({
threadVars[name] = computed[name as keyof ThreadTemplateVars];
}
}
if (runtimeArgs.trace && templateUsesTrajectoryVariables(promptTemplate)) {
if (!runtimeArgs.trace.getTrajectory) {
throw new Error("Trajectory variables need trace.getTrajectory()");
}
Object.assign(
threadVars,
computeTrajectoryTemplateVars(await runtimeArgs.trace.getTrajectory()),
);
}

const prompt =
promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX);
Expand Down
88 changes: 88 additions & 0 deletions js/trajectory-utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/**
* Renders `trace.getTrajectory()` as plain text for LLM-as-a-judge templates.
* `{{trajectory}}` omits the agent's instructions; `{{trajectory_with_system}}` includes them.
*/
/* eslint-disable @typescript-eslint/no-explicit-any */

export function templateUsesTrajectoryVariables(template: string): boolean {
return /\{\{\s*trajectory(_with_system)?\s*\}\}/.test(template);
}

export function computeTrajectoryTemplateVars(trajectories: any[]) {
return {
trajectory: render(trajectories, false),
trajectory_with_system: render(trajectories, true),
};
}

// One block per trajectory, separated by a blank line.
function render(trajectories: any[], withSystem: boolean): string {
return trajectories
.map((t) => renderTurns(t.turns ?? [], withSystem))
.filter(Boolean)
.join("\n\n");
}

function renderTurns(turns: any[], withSystem: boolean): string {
const lines: string[] = [];
let lastSystem = "";
for (const turn of turns) {
const request: any[] = turn.request ?? [];
const work: any[] = turn.work ?? [];

// Some traces keep the instructions only in the LLM call. They can change between turns.
const llmMessages = work
.filter((s) => s.work?.type === "llm_analysis")
.flatMap((s) => s.work.work ?? []);
const systemTexts = [...request, ...llmMessages]
.filter((m) => m.role === "system")
.map((m) => text(m.content));
const system = [...new Set(systemTexts)].join("\n");
if (withSystem && system && system !== lastSystem) {
lines.push(`System: ${system}`);
lastSystem = system;
}

for (const m of request) {
if (m.role === "user") lines.push(`User: ${text(m.content)}`);
}

for (const step of work) {
// Skip steps that are not tool calls, such as span_type "task".
if (step.span_type !== "tool" || step.work?.type !== "tool_result") {
continue;
}
// When the tool call has an ID, Lingua moves the output into `content`.
const part = step.work.content?.find(
(p: any) => p.type === "tool_result",
);
const output = part ? part.output : step.work.output;
const result =
step.error != null ? `error: ${json(step.error)}` : json(output);
const name = step.name || "tool";
lines.push(`Tool: ${name}(${json(step.work.input)}) -> ${result}`);
}

// An interrupted response is already cut off where it stopped.
let response = text(turn.response?.response);
if (turn.interrupted) response += " [interrupted]";
if (response.trim()) lines.push(`Assistant: ${response.trim()}`);
}
return lines.join("\n");
}

// Message content is a string or a list of parts. Only text renders; audio and images don't.
function text(content: any): string {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
return content
.filter((p) => p?.type === "text")
.map((p) => p.text)
.join("\n");
}

// Tool inputs, outputs and errors: strings as they are, anything else as compact JSON.
function json(value: any): string {
if (value === undefined || value === null) return "";
return typeof value === "string" ? value : JSON.stringify(value);
}
10 changes: 8 additions & 2 deletions py/autoevals/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
filter_system_messages_from_thread,
template_uses_thread_variables,
)
from .trajectory_utils import compute_trajectory_template_vars, template_uses_trajectory_variables

# Disable HTML escaping in chevron.
chevron.renderer._html_escape = lambda x: x # type: ignore[attr-defined]
Expand Down Expand Up @@ -378,6 +379,7 @@ def __init__(
**extra_render_args,
):
self._template_uses_thread_variables = template_uses_thread_variables(prompt_template)
self._template_uses_trajectory_variables = template_uses_trajectory_variables(prompt_template)
choice_strings = list(choice_scores.keys())
# Use configured default model if not specified
if model is None:
Expand Down Expand Up @@ -458,18 +460,22 @@ def _request_args(self, output, expected, **kwargs):
thread_vars: dict[str, object] = {}
if trace is not None and self._template_uses_thread_variables:
thread_vars = self._compute_thread_vars_sync(trace)
if trace is not None and self._template_uses_trajectory_variables:
thread_vars.update(compute_trajectory_template_vars(asyncio.run(trace.get_trajectory())))

# Thread vars come first so explicit render args can override.
return super()._request_args(output, expected, **thread_vars, **kwargs)
return super()._request_args(output, expected, **{**thread_vars, **kwargs})

async def _request_args_async(self, output, expected, **kwargs):
trace = kwargs.get("trace")
thread_vars: dict[str, object] = {}
if trace is not None and self._template_uses_thread_variables:
thread_vars = await self._compute_thread_vars_async(trace)
if trace is not None and self._template_uses_trajectory_variables:
thread_vars.update(compute_trajectory_template_vars(await trace.get_trajectory()))

# Thread vars come first so explicit render args can override.
return super()._request_args(output, expected, **thread_vars, **kwargs)
return super()._request_args(output, expected, **{**thread_vars, **kwargs})

@classmethod
def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = None, **kwargs):
Expand Down
82 changes: 82 additions & 0 deletions py/autoevals/trajectory_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
"""Renders `trace.get_trajectory()` as plain text for LLM-as-a-judge templates.

`{{trajectory}}` omits the agent's instructions; `{{trajectory_with_system}}` includes them.
"""

import json
import re
from typing import Any


def template_uses_trajectory_variables(template: str) -> bool:
return re.search(r"\{\{\s*trajectory(_with_system)?\s*\}\}", template) is not None


def compute_trajectory_template_vars(trajectories: list[Any]) -> dict[str, str]:
return {
"trajectory": _render(trajectories, with_system=False),
"trajectory_with_system": _render(trajectories, with_system=True),
}


# One block per trajectory, separated by a blank line.
def _render(trajectories: list[Any], with_system: bool) -> str:
blocks = [_render_turns(t.get("turns") or [], with_system) for t in trajectories]
return "\n\n".join(b for b in blocks if b)


def _render_turns(turns: list[Any], with_system: bool) -> str:
lines: list[str] = []
last_system = ""
for turn in turns:
request = turn.get("request") or []
work = turn.get("work") or []

# Some traces keep the instructions only in the LLM call. They can change between turns.
llm_messages = [
m for s in work if (s.get("work") or {}).get("type") == "llm_analysis" for m in s["work"].get("work") or []
]
system_texts = [_text(m.get("content")) for m in request + llm_messages if m.get("role") == "system"]
system = "\n".join(dict.fromkeys(system_texts))
if with_system and system and system != last_system:
lines.append(f"System: {system}")
last_system = system

lines += [f"User: {_text(m.get('content'))}" for m in request if m.get("role") == "user"]

for step in work:
# Skip steps that are not tool calls, such as span_type "task".
step_work = step.get("work") or {}
if step.get("span_type") != "tool" or step_work.get("type") != "tool_result":
continue
# When the tool call has an ID, Lingua moves the output into `content`.
parts = [p for p in step_work.get("content") or [] if p.get("type") == "tool_result"]
output = parts[0].get("output") if parts else step_work.get("output")
error = step.get("error")
result = f"error: {_json(error)}" if error is not None else _json(output)
name = step.get("name") or "tool"
lines.append(f"Tool: {name}({_json(step_work.get('input'))}) -> {result}")

# An interrupted response is already cut off where it stopped.
response = _text((turn.get("response") or {}).get("response"))
if turn.get("interrupted"):
response += " [interrupted]"
if response.strip():
lines.append(f"Assistant: {response.strip()}")
return "\n".join(lines)


# Message content is a string or a list of parts. Only text renders; audio and images don't.
def _text(content: Any) -> str:
if isinstance(content, str):
return content
if not isinstance(content, list):
return ""
return "\n".join(p["text"] for p in content if isinstance(p, dict) and p.get("type") == "text")


# Tool inputs, outputs and errors: strings as they are, anything else as compact JSON.
def _json(value: Any) -> str:
if value is None:
return ""
return value if isinstance(value, str) else json.dumps(value, separators=(",", ":"), ensure_ascii=False)
Loading