From 22896b4cd99951dab88b457e9599b7d25a25278a Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:10:58 +0800 Subject: [PATCH 1/2] FEAT: add raise-on-empty variants to true/false score aggregators --- .../true_false/true_false_score_aggregator.py | 31 ++++++++++++++++ .../score/test_true_false_score_aggregator.py | 37 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/pyrit/score/true_false/true_false_score_aggregator.py b/pyrit/score/true_false/true_false_score_aggregator.py index 7e069dba3a..9e7178be50 100644 --- a/pyrit/score/true_false/true_false_score_aggregator.py +++ b/pyrit/score/true_false/true_false_score_aggregator.py @@ -97,6 +97,7 @@ def _create_aggregator( result_func: Callable[[list[bool | None]], bool | None], true_msg: str, false_msg: str, + raise_on_empty: bool = False, ) -> TrueFalseAggregatorFunc: """ Create a True/False aggregator using a result function over boolean values. @@ -108,6 +109,7 @@ def _create_aggregator( constituent scores, and a ``None`` result means no verdict was reachable. true_msg (str): Description to use when the result is True. false_msg (str): Description to use when the result is False. + raise_on_empty (bool): Whether to raise ValueError when no scores are provided. Defaults to False. Returns: TrueFalseAggregatorFunc: Aggregator function that reduces a sequence of true/false Scores @@ -124,6 +126,8 @@ def aggregator(scores: Iterable[Score]) -> ScoreAggregatorResult: raise ValueError("All scores must be of type 'true_false'.") if not scores_list: + if raise_on_empty: + raise ValueError("No scores available for aggregation") # No scores; return a neutral result return ScoreAggregatorResult( value=False, @@ -162,6 +166,7 @@ def _create_binary_aggregator( op: BinaryBoolOp, true_msg: str, false_msg: str, + raise_on_empty: bool = False, ) -> TrueFalseAggregatorFunc: """ Turn a binary operator over verdicts (e.g. ``_and``) into an aggregation function. @@ -171,6 +176,7 @@ def _create_binary_aggregator( op (BinaryBoolOp): Binary three-valued operator to apply. true_msg (str): Description to use when the result is True. false_msg (str): Description to use when the result is False. + raise_on_empty (bool): Whether to raise ValueError when no scores are provided. Defaults to False. Returns: TrueFalseAggregatorFunc: Aggregator function that reduces scores using the binary operator. @@ -180,6 +186,7 @@ def _create_binary_aggregator( result_func=lambda bs, _op=op: functools.reduce(_op, bs), true_msg=true_msg, false_msg=false_msg, + raise_on_empty=raise_on_empty, ) @@ -224,3 +231,27 @@ class TrueFalseScoreAggregator: true_msg="A strict majority of constituent scorers returned True in a MAJORITY composite scorer.", false_msg="A strict majority of constituent scorers did not return True in a MAJORITY composite scorer.", ) + + AND_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_binary_aggregator( + "AND_RAISE_ON_EMPTY", + _and, + "All constituent scorers returned True in an AND composite scorer.", + "At least one constituent scorer returned False in an AND composite scorer.", + raise_on_empty=True, + ) + + OR_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_binary_aggregator( + "OR_RAISE_ON_EMPTY", + _or, + "At least one constituent scorer returned True in an OR composite scorer.", + "All constituent scorers returned False in an OR composite scorer.", + raise_on_empty=True, + ) + + MAJORITY_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_aggregator( + "MAJORITY_RAISE_ON_EMPTY", + result_func=_majority, + true_msg="A strict majority of constituent scorers returned True in a MAJORITY composite scorer.", + false_msg="A strict majority of constituent scorers did not return True in a MAJORITY composite scorer.", + raise_on_empty=True, + ) diff --git a/tests/unit/score/test_true_false_score_aggregator.py b/tests/unit/score/test_true_false_score_aggregator.py index bb5f16dec9..db1109582f 100644 --- a/tests/unit/score/test_true_false_score_aggregator.py +++ b/tests/unit/score/test_true_false_score_aggregator.py @@ -276,3 +276,40 @@ def test_generator_of_wrong_type_still_raises(): ) with pytest.raises(ValueError, match="must be of type 'true_false'"): TrueFalseScoreAggregator.OR(s for s in [bad]) + + +# Tests for raise_on_empty behavior +def test_and_raise_on_empty_with_scores(): + """Test that AND_RAISE_ON_EMPTY works normally when scores are present.""" + scores = [_mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")] + res = TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY(scores) + assert res.value is False + + +def test_or_raise_on_empty_with_scores(): + """Test that OR_RAISE_ON_EMPTY works normally when scores are present.""" + scores = [_mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")] + res = TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY(scores) + assert res.value is True + + +def test_majority_raise_on_empty_with_scores(): + """Test that MAJORITY_RAISE_ON_EMPTY works normally when scores are present.""" + scores = [_mk_score(True, prr_id="1"), _mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")] + res = TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY(scores) + assert res.value is True + + +@pytest.mark.parametrize("aggregator_name", ["AND_RAISE_ON_EMPTY", "OR_RAISE_ON_EMPTY", "MAJORITY_RAISE_ON_EMPTY"]) +def test_raise_on_empty_aggregators_raise_with_no_scores(aggregator_name): + """The strict-empty variants must not answer an empty input with a verdict.""" + aggregator = getattr(TrueFalseScoreAggregator, aggregator_name) + with pytest.raises(ValueError, match="No scores available for aggregation"): + aggregator([]) + + +def test_raise_on_empty_aggregator_accepts_generators(): + """A generator with scores must not trip the empty-input guard.""" + values = [True, False, True] + res = TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY(_mk_score(v, prr_id="1") for v in values) + assert res.value is True From 61682d92e2afe85bf18aa0c6c4a98e02187c6b8a Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 9 Oct 2026 13:14:55 -0700 Subject: [PATCH 2/2] FIX: preserve strict composite evaluation identity across child reorderings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../true_false/true_false_composite_scorer.py | 8 ++- .../test_scorer_evaluation_identifier.py | 52 +++++++++++++++---- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 2cf604c09e..19ef2852f4 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -39,8 +39,9 @@ class TrueFalseCompositeScorer(TrueFalseScorer): Children are true/false scorers of any evidence kind, so a scorer over a message can be composed with one over evidence that is not a message at all. - Built-in AND, OR, and MAJORITY aggregators opt into order-independent evaluation - identity. Duplicates remain significant; custom aggregators and execution stay ordered. + Built-in AND, OR, and MAJORITY aggregators, including their raise-on-empty variants, + opt into order-independent evaluation identity. Duplicates remain significant; + custom aggregators and execution stay ordered. """ def __init__( @@ -87,6 +88,9 @@ def _build_identifier(self) -> ComponentIdentifier: TrueFalseScoreAggregator.AND, TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.MAJORITY, + TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, ) ) return self._create_identifier( diff --git a/tests/unit/score/test_scorer_evaluation_identifier.py b/tests/unit/score/test_scorer_evaluation_identifier.py index eb094f2d77..b3b8b2f89f 100644 --- a/tests/unit/score/test_scorer_evaluation_identifier.py +++ b/tests/unit/score/test_scorer_evaluation_identifier.py @@ -101,7 +101,14 @@ def test_eval_hash_matches_free_function(self): class TestCompositeEvaluationOrder: @pytest.mark.parametrize( "aggregator", - [TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND, TrueFalseScoreAggregator.MAJORITY], + [ + TrueFalseScoreAggregator.OR, + TrueFalseScoreAggregator.AND, + TrueFalseScoreAggregator.MAJORITY, + TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, + ], ) def test_permutations_preserve_eval_identity_and_content_order(self, aggregator: TrueFalseAggregatorFunc) -> None: children = [SubStringScorer(substring=value) for value in ("a", "b", "c")] @@ -120,15 +127,25 @@ def test_permutations_preserve_eval_identity_and_content_order(self, aggregator: assert restored.hash == identifier.hash assert ScorerEvaluationIdentifier(restored).eval_hash == identifier.eval_hash - def test_nested_permutations_preserve_eval_identity(self) -> None: + @pytest.mark.parametrize( + "outer_aggregator, inner_aggregator", + [ + (TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND), + (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY), + (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY), + ], + ) + def test_nested_permutations_preserve_eval_identity( + self, *, outer_aggregator: TrueFalseAggregatorFunc, inner_aggregator: TrueFalseAggregatorFunc + ) -> None: a, b, c = [SubStringScorer(substring=value) for value in ("a", "b", "c")] first = TrueFalseCompositeScorer( - aggregator=TrueFalseScoreAggregator.OR, - scorers=[TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[a, b]), c], + aggregator=outer_aggregator, + scorers=[TrueFalseCompositeScorer(aggregator=inner_aggregator, scorers=[a, b]), c], ) second = TrueFalseCompositeScorer( - aggregator=TrueFalseScoreAggregator.OR, - scorers=[c, TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[b, a])], + aggregator=outer_aggregator, + scorers=[c, TrueFalseCompositeScorer(aggregator=inner_aggregator, scorers=[b, a])], ) assert first.get_identifier().eval_hash == second.get_identifier().eval_hash assert first.get_identifier().hash != second.get_identifier().hash @@ -142,6 +159,12 @@ def test_configuration_aggregator_and_multiplicity_remain_distinct(self) -> None (TrueFalseScoreAggregator.MAJORITY, [a, b]), (TrueFalseScoreAggregator.MAJORITY, [a, a, b]), (TrueFalseScoreAggregator.MAJORITY, [a, b, b]), + (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, [a, b]), + (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, [a, c]), + (TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY, [a, b]), + (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, b]), + (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, a, b]), + (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, b, b]), ] hashes = { TrueFalseCompositeScorer(aggregator=aggregator, scorers=scorers).get_identifier().eval_hash @@ -149,15 +172,26 @@ def test_configuration_aggregator_and_multiplicity_remain_distinct(self) -> None } assert len(hashes) == len(configurations) - def test_custom_aggregator_with_builtin_name_remains_ordered(self) -> None: + @pytest.mark.parametrize( + "aggregator", + [ + TrueFalseScoreAggregator.OR, + TrueFalseScoreAggregator.AND, + TrueFalseScoreAggregator.MAJORITY, + TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY, + TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, + ], + ) + def test_custom_aggregator_with_builtin_name_remains_ordered(self, aggregator: TrueFalseAggregatorFunc) -> None: def first_score(scores: Iterable[Score]) -> ScoreAggregatorResult: return TrueFalseScoreAggregator.OR([next(iter(scores))]) - first_score.__name__ = TrueFalseScoreAggregator.OR.__name__ + first_score.__name__ = aggregator.__name__ a, b = [SubStringScorer(substring=value) for value in ("a", "b")] first = TrueFalseCompositeScorer(aggregator=first_score, scorers=[a, b]).get_identifier() second = TrueFalseCompositeScorer(aggregator=first_score, scorers=[b, a]).get_identifier() - builtin = TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.OR, scorers=[a, b]).get_identifier() + builtin = TrueFalseCompositeScorer(aggregator=aggregator, scorers=[a, b]).get_identifier() assert "sub_scorers_order_independent" not in first.params assert first.eval_hash != second.eval_hash