diff --git a/js/ragas.test.ts b/js/ragas.test.ts index 9a2aaa3..8eac07e 100644 --- a/js/ragas.test.ts +++ b/js/ragas.test.ts @@ -13,6 +13,7 @@ import { Faithfulness, } from "./ragas"; import { init } from "./oai"; +import { renderMessages } from "./render-messages"; const data = { input: "Can starred docs from different workspaces be accessed in one place?", @@ -327,3 +328,167 @@ describe("AnswerCorrectness custom embedding model", () => { expect(capturedEmbeddingModel).toBe("text-embedding-3-large"); }); }); + +// Mustache HTML-escapes `{{...}}` interpolations by default, so a value +// containing <, >, &, " or / used to reach the judge model as HTML entities +// instead of the text the caller supplied. Every other prompt path in the +// library (`render-messages`, used by the LLMClassifier scorers) already +// renders values verbatim; the ragas scorers must send the same text. +describe("Ragas prompt rendering", () => { + const server = setupServer(); + + // Contains every character mustache escapes: < > & " and /. + const TEXT = 'Is 3 < 5 & 2 > 1? "q"'; + const ESCAPED = ["<", ">", "&", """, "/"]; + + let prompts: string[] = []; + + beforeAll(() => { + server.listen({ + onUnhandledRequest: (req) => { + throw new Error(`Unhandled request ${req.method}, ${req.url}`); + }, + }); + }); + + afterEach(() => { + server.resetHandlers(); + prompts = []; + init(); + }); + + afterAll(() => { + server.close(); + }); + + // Records every user message sent to the model and answers with a tool call + // for whichever tool the scorer asked for. + const capturePrompts = ( + toolArgs: (toolName: string) => Record, + ) => { + server.use( + http.post( + "https://api.openai.com/v1/chat/completions", + async ({ request }) => { + const body = (await request.json()) as { + messages: { role: string; content: string }[]; + tools?: { function: { name: string } }[]; + }; + prompts.push(body.messages[0].content); + const name = body.tools?.[0]?.function.name ?? ""; + return HttpResponse.json({ + id: "chatcmpl-test", + object: "chat.completion", + created: Date.now(), + model: "gpt-5-mini", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_test", + type: "function", + function: { + name, + arguments: JSON.stringify(toolArgs(name)), + }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + }, + }); + }, + ), + ); + + init({ + client: new OpenAI({ + apiKey: "test-api-key", + baseURL: "https://api.openai.com/v1", + }), + }); + }; + + const expectVerbatim = (prompt: string) => { + expect(prompt).toContain(TEXT); + for (const entity of ESCAPED) { + expect(prompt).not.toContain(entity); + } + }; + + test("ContextEntityRecall sends `text` to the model verbatim", async () => { + capturePrompts(() => ({ entities: ["5"] })); + + await ContextEntityRecall({ + input: "Is three less than five?", + output: "5", + expected: TEXT, + context: TEXT, + // Avoid the embeddings endpoint; the entity lists match either way. + pairwiseScorer: async () => ({ name: "exact", score: 1 }), + }); + + // Once for `expected`, once for `context`. + expect(prompts).toHaveLength(2); + for (const prompt of prompts) { + expectVerbatim(prompt); + } + }); + + test("ContextRelevancy sends `question` and `context` verbatim", async () => { + capturePrompts(() => ({ sentences: [] })); + + await ContextRelevancy({ + input: TEXT, + output: "5", + context: TEXT, + }); + + expect(prompts).toHaveLength(1); + // Both are interpolated into the one prompt. + expect(prompts[0]).toContain(`question: ${TEXT}`); + expect(prompts[0]).toContain(`context: ${TEXT}`); + }); + + test("Faithfulness sends `context` verbatim and `statements` as JSON", async () => { + capturePrompts((name) => + name === "extract_statements" + ? { statements: ["The answer is 5"] } + : { faithfulness: [{ statement: "x", verdict: 1, reason: "y" }] }, + ); + + await Faithfulness({ + input: "Is three less than five?", + output: "5", + context: TEXT, + }); + + // First call extracts statements from the answer, second judges them. + expect(prompts).toHaveLength(2); + expectVerbatim(prompts[1]); + // `statements` is a string[]; render-messages stringifies non-strings as + // JSON, and the scorers follow the same rule. + expect(prompts[1]).toContain( + `statements: ${JSON.stringify(["The answer is 5"])}`, + ); + }); + + test("agrees with renderMessages, the library's other prompt path", () => { + const rendered = renderMessages( + [{ role: "user", content: "text: {{text}}" }], + { text: TEXT }, + )[0].content; + + expect(rendered).toBe(`text: ${TEXT}`); + }); +}); diff --git a/js/ragas.ts b/js/ragas.ts index e961460..ecd0b78 100644 --- a/js/ragas.ts +++ b/js/ragas.ts @@ -111,6 +111,7 @@ import { ListContains } from "./list"; import { EmbeddingSimilarity } from "./string"; import { z } from "zod/v3"; import { makePartial, ScorerWithPartial } from "./partial"; +import { escapeValue } from "./render-messages"; type RagasArgs = { input?: string; @@ -184,7 +185,9 @@ export const ContextEntityRecall: ScorerWithPartial< messages: [ { role: "user", - content: mustache.render(ENTITY_PROMPT, { text }), + content: mustache.render(ENTITY_PROMPT, { text }, undefined, { + escape: escapeValue, + }), }, ], tools: [ @@ -258,10 +261,15 @@ export const ContextRelevancy: ScorerWithPartial = messages: [ { role: "user", - content: mustache.render(SENTENCE_PROMPT, { - question: input, - context, - }), + content: mustache.render( + SENTENCE_PROMPT, + { + question: input, + context, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -357,11 +365,16 @@ export const ContextRecall: ScorerWithPartial = makePartial( messages: [ { role: "user", - content: mustache.render(CONTEXT_RECALL_PROMPT, { - question: input, - answer: expected, - context, - }), + content: mustache.render( + CONTEXT_RECALL_PROMPT, + { + question: input, + answer: expected, + context, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -456,11 +469,16 @@ export const ContextPrecision: ScorerWithPartial = messages: [ { role: "user", - content: mustache.render(CONTEXT_PRECISION_PROMPT, { - question: input, - answer: expected, - context, - }), + content: mustache.render( + CONTEXT_PRECISION_PROMPT, + { + question: input, + answer: expected, + context, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -580,10 +598,15 @@ export const Faithfulness: ScorerWithPartial = makePartial( messages: [ { role: "user", - content: mustache.render(LONG_FORM_ANSWER_PROMPT, { - question: input, - answer: output, - }), + content: mustache.render( + LONG_FORM_ANSWER_PROMPT, + { + question: input, + answer: output, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -608,10 +631,15 @@ export const Faithfulness: ScorerWithPartial = makePartial( messages: [ { role: "user", - content: mustache.render(NLI_STATEMENTS_PROMPT, { - context, - statements, - }), + content: mustache.render( + NLI_STATEMENTS_PROMPT, + { + context, + statements, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -715,10 +743,15 @@ export const AnswerRelevancy: ScorerWithPartial< messages: [ { role: "user", - content: mustache.render(QUESTION_GEN_PROMPT, { - answer: output, - context, - }), + content: mustache.render( + QUESTION_GEN_PROMPT, + { + answer: output, + context, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ @@ -890,11 +923,16 @@ export const AnswerCorrectness: ScorerWithPartial< messages: [ { role: "user", - content: mustache.render(CORRECTNESS_PROMPT, { - question: input, - answer: output, - ground_truth: expected, - }), + content: mustache.render( + CORRECTNESS_PROMPT, + { + question: input, + answer: output, + ground_truth: expected, + }, + undefined, + { escape: escapeValue }, + ), }, ], tools: [ diff --git a/js/render-messages.ts b/js/render-messages.ts index 73cb1ec..4c87030 100644 --- a/js/render-messages.ts +++ b/js/render-messages.ts @@ -13,7 +13,7 @@ import { * - Single messages are formatted with role and content * - Other values are JSON-stringified */ -function escapeValue(v: unknown): string { +export function escapeValue(v: unknown): string { if (typeof v === "string") { return v; }