Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions py/autoevals/partial.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
from .score import Scorer

_UNSET = object()


class ScorerWithPartial(Scorer):
@classmethod
def partial(cls, **partial_kwargs):
class PartialScorer(cls):
async def eval_async(self, output, expected=None, **kwargs):
if expected is not None:
async def eval_async(self, output, expected=_UNSET, **kwargs):
if expected is not _UNSET:
kwargs["expected"] = expected
return await self._run_eval_async(output, **{**partial_kwargs, **kwargs})

def eval(self, output, expected=None, **kwargs):
if expected is not None:
def eval(self, output, expected=_UNSET, **kwargs):
if expected is not _UNSET:
kwargs["expected"] = expected
return self._run_eval_sync(output, **{**partial_kwargs, **kwargs})

def __call__(self, output, expected=_UNSET, **kwargs):
return self.eval(output, expected, **kwargs)

@classmethod
def _partial_args(cls):
return {**partial_kwargs}
Expand Down
51 changes: 51 additions & 0 deletions py/autoevals/test_partial.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import asyncio

import pytest

from autoevals.llm import ClosedQA
from autoevals.string import Levenshtein
from autoevals.value import ExactMatch


def test_partial():
Expand All @@ -14,3 +19,49 @@ def test_partial():
assert closedqa_partial.score == closedqa_basic.score
assert closedqa_partial.name == closedqa_basic.name
assert closedqa_partial.name == "ClosedQA"


def run_partial(scorer, method, *args, **kwargs):
result = getattr(scorer, method)(*args, **kwargs)
return asyncio.run(result) if method == "eval_async" else result


@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"])
@pytest.mark.parametrize("output", ["approved", None])
@pytest.mark.parametrize("positional", [False, True])
def test_partial_explicit_none_overrides_bound_expected(method, output, positional):
scorer = ExactMatch.partial(expected="approved")()
if positional:
result = run_partial(scorer, method, output, None)
else:
result = run_partial(scorer, method, output=output, expected=None)

assert result.score == ExactMatch().eval(output, expected=None).score
assert result.name == "ExactMatch"
assert run_partial(scorer, method, output="approved").score == 1


@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"])
def test_partial_omitted_expected_retains_bound_value(method):
scorer = ExactMatch.partial(expected="approved")()
assert run_partial(scorer, method, output="approved").score == 1


@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"])
@pytest.mark.parametrize("expected", ["rejected", "", 0, False])
def test_partial_non_null_overrides_remain_supported(method, expected):
scorer = ExactMatch.partial(expected="approved")()
assert run_partial(scorer, method, output=expected, expected=expected).score == 1


@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"])
def test_partial_without_bound_expected_keeps_default_none(method):
scorer = ExactMatch.partial()()
assert run_partial(scorer, method, output=None).score == 1


@pytest.mark.parametrize("method", ["eval", "eval_async", "__call__"])
def test_partial_explicit_none_preserves_required_expected_validation(method):
scorer = Levenshtein.partial(expected="approved")()
with pytest.raises(ValueError, match="requires an expected value"):
run_partial(scorer, method, output="approved", expected=None)