Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion py/autoevals/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
)

COT_SUFFIX = """\
Answer the question by calling `select_choice` with your reasoning in a step-by-step matter to be
Answer the question by calling `select_choice` with your reasoning in a step-by-step manner to be
sure that your conclusion is correct. Avoid simply stating the correct answer at the outset. Select a
single choice by setting the `choice` parameter to a single choice from {{__choices}}.
""".strip().replace(
Expand Down
8 changes: 8 additions & 0 deletions py/autoevals/test_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,14 @@ async def test_llm_classifier_request_args_async_keep_thread_filtered_and_thread
assert system_marker in full_thread


def test_cot_suffix_wording_matches_js():
request_args = Factuality()._request_args(input="question", output="answer", expected="answer")
rendered_prompt = request_args["messages"][0]["content"]

assert "with your reasoning in a step-by-step manner to be sure" in rendered_prompt
assert "step-by-step matter" not in rendered_prompt


def test_openai():
e = OpenAILLMClassifier(
"title",
Expand Down