From dee91cba80d4d46621bc848e59912a4a1b506699 Mon Sep 17 00:00:00 2001 From: Abhay Mettu <113219143+abhaymettu@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:35:19 -0600 Subject: [PATCH] fix(py): CoT suffix says step-by-step manner, as in JS The Python COT_SUFFIX still read "step-by-step matter", a typo #66 fixed only in js/llm.ts, so Python and JS judges sent different prompts. Pin the corrected wording with a test on the rendered Factuality prompt. --- py/autoevals/llm.py | 2 +- py/autoevals/test_llm.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 1a38712..bfd8edb 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -80,7 +80,7 @@ ) COT_SUFFIX = """\ -Answer the question by calling `select_choice` with your reasoning in a step-by-step matter to be +Answer the question by calling `select_choice` with your reasoning in a step-by-step manner to be sure that your conclusion is correct. Avoid simply stating the correct answer at the outset. Select a single choice by setting the `choice` parameter to a single choice from {{__choices}}. """.strip().replace( diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 2bcecef..6b2dc70 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -188,6 +188,14 @@ async def test_llm_classifier_request_args_async_keep_thread_filtered_and_thread assert system_marker in full_thread +def test_cot_suffix_wording_matches_js(): + request_args = Factuality()._request_args(input="question", output="answer", expected="answer") + rendered_prompt = request_args["messages"][0]["content"] + + assert "with your reasoning in a step-by-step manner to be sure" in rendered_prompt + assert "step-by-step matter" not in rendered_prompt + + def test_openai(): e = OpenAILLMClassifier( "title",