From ef848e7065cdf1b5e4611732ab6d0a9a366a6b94 Mon Sep 17 00:00:00 2001 From: vinsymoyya <186405436+vinsymoyya@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:55:43 +0530 Subject: [PATCH] fix(python): honor explicit None in partial scorers --- py/autoevals/partial.py | 13 ++++++--- py/autoevals/test_partial.py | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/py/autoevals/partial.py b/py/autoevals/partial.py index 290b6be4..9f180db6 100644 --- a/py/autoevals/partial.py +++ b/py/autoevals/partial.py @@ -1,20 +1,25 @@ from .score import Scorer +_UNSET = object() + class ScorerWithPartial(Scorer): @classmethod def partial(cls, **partial_kwargs): class PartialScorer(cls): - async def eval_async(self, output, expected=None, **kwargs): - if expected is not None: + async def eval_async(self, output, expected=_UNSET, **kwargs): + if expected is not _UNSET: kwargs["expected"] = expected return await self._run_eval_async(output, **{**partial_kwargs, **kwargs}) - def eval(self, output, expected=None, **kwargs): - if expected is not None: + def eval(self, output, expected=_UNSET, **kwargs): + if expected is not _UNSET: kwargs["expected"] = expected return self._run_eval_sync(output, **{**partial_kwargs, **kwargs}) + def __call__(self, output, expected=_UNSET, **kwargs): + return self.eval(output, expected, **kwargs) + @classmethod def _partial_args(cls): return {**partial_kwargs} diff --git a/py/autoevals/test_partial.py b/py/autoevals/test_partial.py index 1604d11e..ee2e6d41 100644 --- a/py/autoevals/test_partial.py +++ b/py/autoevals/test_partial.py @@ -1,5 +1,10 @@ +import asyncio + +import pytest + from autoevals.llm import ClosedQA from autoevals.string import Levenshtein +from autoevals.value import ExactMatch def test_partial(): @@ -14,3 +19,49 @@ def test_partial(): assert closedqa_partial.score == closedqa_basic.score assert closedqa_partial.name == closedqa_basic.name assert closedqa_partial.name == "ClosedQA" + + +def run_partial(scorer, method, *args, **kwargs): + result = getattr(scorer, method)(*args, **kwargs) + return asyncio.run(result) if method == "eval_async" else result + + +@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"]) +@pytest.mark.parametrize("output", ["approved", None]) +@pytest.mark.parametrize("positional", [False, True]) +def test_partial_explicit_none_overrides_bound_expected(method, output, positional): + scorer = ExactMatch.partial(expected="approved")() + if positional: + result = run_partial(scorer, method, output, None) + else: + result = run_partial(scorer, method, output=output, expected=None) + + assert result.score == ExactMatch().eval(output, expected=None).score + assert result.name == "ExactMatch" + assert run_partial(scorer, method, output="approved").score == 1 + + +@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"]) +def test_partial_omitted_expected_retains_bound_value(method): + scorer = ExactMatch.partial(expected="approved")() + assert run_partial(scorer, method, output="approved").score == 1 + + +@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"]) +@pytest.mark.parametrize("expected", ["rejected", "", 0, False]) +def test_partial_non_null_overrides_remain_supported(method, expected): + scorer = ExactMatch.partial(expected="approved")() + assert run_partial(scorer, method, output=expected, expected=expected).score == 1 + + +@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"]) +def test_partial_without_bound_expected_keeps_default_none(method): + scorer = ExactMatch.partial()() + assert run_partial(scorer, method, output=None).score == 1 + + +@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"]) +def test_partial_explicit_none_preserves_required_expected_validation(method): + scorer = Levenshtein.partial(expected="approved")() + with pytest.raises(ValueError, match="requires an expected value"): + run_partial(scorer, method, output="approved", expected=None)