diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 1a38712..92886ae 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -485,7 +485,8 @@ def from_spec(cls, name: str, spec: ModelGradedSpec, client: Client | None = Non if spec.max_tokens is not None: spec_kwargs["max_tokens"] = spec.max_tokens # kwargs can override spec values - return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs, **kwargs) + spec_kwargs.update(kwargs) + return cls(name, spec.prompt, spec.choice_scores, client=client, **spec_kwargs) @classmethod def from_spec_file(cls, name: str, path: str, client: Client | None = None, **kwargs): diff --git a/py/autoevals/test_llm.py b/py/autoevals/test_llm.py index 2bcecef..208cb8b 100644 --- a/py/autoevals/test_llm.py +++ b/py/autoevals/test_llm.py @@ -14,6 +14,7 @@ Battle, Factuality, LLMClassifier, + ModelGradedSpec, OpenAILLMClassifier, build_classification_tools, ) @@ -778,3 +779,63 @@ async def get_thread(self): classifier.eval(output="x", expected="y", trace=trace) assert trace.calls == 0 + + +@pytest.mark.parametrize( + "option,saved,override", + [ + ("model", "saved-model", "requested-model"), + ("engine", "saved-engine", "requested-engine"), + ("use_cot", True, False), + ("temperature", 0.7, 0.0), + ("max_tokens", 50, 17), + ], +) +def test_from_spec_explicit_options_override_saved_values(option, saved, override): + spec = ModelGradedSpec(prompt="Grade {{output}}", choice_scores={"A": 1, "B": 0}, **{option: saved}) + classifier = LLMClassifier.from_spec("fixture-spec", spec, **{option: override}) + if option == "use_cot": + assert "reasons" not in classifier.classification_tools[0]["function"]["parameters"]["properties"] + elif option in {"temperature", "max_tokens"}: + assert classifier.extra_args[option] == override + else: + assert getattr(classifier, option) == override + assert getattr(spec, option) == saved + + +def test_from_spec_preserves_saved_defaults_and_other_arguments(): + client = OpenAI(api_key="fixture-key") + spec = ModelGradedSpec( + prompt="Grade {{output}} with {{rubric}}", + choice_scores={"A": 1, "B": 0}, + model="saved-model", + temperature=0.7, + max_tokens=50, + use_cot=False, + ) + unchanged = LLMClassifier.from_spec("fixture-unchanged", spec, client=client, rubric="concise") + assert unchanged.model == "saved-model" + assert unchanged.extra_args == {"temperature": 0.7, "max_tokens": 50} + assert unchanged.client is client + assert unchanged.render_args["rubric"] == "concise" + changed = LLMClassifier.from_spec( + "fixture-mixed", spec, client=client, model="requested-model", temperature=0, max_tokens=None + ) + assert changed.model == "requested-model" + assert changed.extra_args == {"temperature": 0} + assert changed.choice_scores == spec.choice_scores + assert changed.client is client + + +def test_from_spec_file_allows_saved_option_overrides(tmp_path): + path = tmp_path / "fixture.yaml" + path.write_text( + 'prompt: "Grade {{output}}"\nchoice_scores: {"A": 1, "B": 0}\nmodel: saved-model\ntemperature: 0.7\n' + ) + name = str(path) + try: + classifier = LLMClassifier.from_spec_file(name, str(path), model="requested-model", temperature=0) + assert classifier.model == "requested-model" + assert classifier.extra_args["temperature"] == 0 + finally: + LLMClassifier._SPEC_FILE_CONTENTS.pop(name, None)