From 1cc3ddc8f7ae3fe89e404de9ff3855c0dd70d476 Mon Sep 17 00:00:00 2001 From: Rohith Pariki Date: Sat, 3 Oct 2026 18:52:05 +0530 Subject: [PATCH 1/2] FIX: validate n-gram size and reference text in PlagiarismScorer (#2971) --- pyrit/score/float_scale/plagiarism_scorer.py | 21 +++++++- tests/unit/score/test_plagiarism_scorer.py | 56 ++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index e4d1a4b1b1..435ad76541 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -48,13 +48,27 @@ def __init__( metric (PlagiarismMetric): The plagiarism detection metric to use. Defaults to PlagiarismMetric.LCS. n (int): The n-gram size for n-gram similarity. Defaults to 5. validator (ScorerPromptValidator | None): Custom validator for the scorer. Defaults to None. + + Raises: + ValueError: If ``reference_text`` is not a non-empty string or contains no word tokens, + if ``metric`` is not an instance of PlagiarismMetric, or if ``n`` is not an integer >= 1. """ - super().__init__(validator=validator or self._DEFAULT_VALIDATOR) + if not isinstance(reference_text, str) or not reference_text.strip(): + raise ValueError("reference_text must be a non-empty string.") + if not isinstance(metric, PlagiarismMetric): + raise ValueError(f"metric must be an instance of PlagiarismMetric, got {metric!r}.") + if not isinstance(n, int) or isinstance(n, bool) or n < 1: + raise ValueError(f"n must be an integer >= 1, got {n!r}.") self.reference_text = reference_text self.metric = metric self.n = n + if not self._tokenize(reference_text): + raise ValueError("reference_text must contain at least one word token.") + + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) + def _build_identifier(self) -> ComponentIdentifier: """ Build the identifier for this scorer. @@ -152,6 +166,11 @@ def _plagiarism_score( metric: PlagiarismMetric = PlagiarismMetric.LCS, n: int = 5, ) -> float: + if not isinstance(n, int) or isinstance(n, bool) or n < 1: + raise ValueError(f"n must be an integer >= 1, got {n!r}.") + if not hasattr(metric, "value"): + raise ValueError("metric must be 'lcs', 'levenshtein', or 'jaccard'") + tokens_response = self._tokenize(response) tokens_reference = self._tokenize(reference) response_len = len(tokens_response) diff --git a/tests/unit/score/test_plagiarism_scorer.py b/tests/unit/score/test_plagiarism_scorer.py index d217fc8cf5..0d8fd6fab9 100644 --- a/tests/unit/score/test_plagiarism_scorer.py +++ b/tests/unit/score/test_plagiarism_scorer.py @@ -36,6 +36,62 @@ def test_init_with_custom_parameters(self): assert scorer.metric == metric assert scorer.n == n + @pytest.mark.parametrize( + "invalid_reference", + ["", " ", "\t\n ", None, 123, [], {}], + ) + def test_init_rejects_empty_or_non_string_reference_text(self, invalid_reference): + """Test initialization rejects empty, whitespace-only, or non-string reference text.""" + with pytest.raises(ValueError, match="reference_text must be a non-empty string"): + PlagiarismScorer(reference_text=invalid_reference) + + @pytest.mark.parametrize( + "no_token_reference", + ["!!!", "???", "---", "... ,,, ;;;", " !@#$%^&*() "], + ) + def test_init_rejects_reference_text_without_tokens(self, no_token_reference): + """Test initialization rejects reference text containing no word tokens.""" + with pytest.raises(ValueError, match="reference_text must contain at least one word token"): + PlagiarismScorer(reference_text=no_token_reference) + + @pytest.mark.parametrize( + "invalid_n", + [0, -1, -5, 1.5, False, True, "3", None, [3]], + ) + def test_init_rejects_invalid_n(self, invalid_n): + """Test initialization rejects n that is not an integer >= 1 or is a boolean.""" + with pytest.raises(ValueError, match=r"n must be an integer >= 1"): + PlagiarismScorer(reference_text="Valid reference text", n=invalid_n) + + @pytest.mark.parametrize("valid_n", [1, 2, 5, 10]) + def test_init_accepts_valid_boundary_n(self, valid_n): + """Test initialization accepts positive integer n-gram sizes.""" + scorer = PlagiarismScorer(reference_text="Valid reference text", n=valid_n) + assert scorer.n == valid_n + + @pytest.mark.parametrize( + "invalid_metric", + ["lcs", "levenshtein", "jaccard", "invalid", None, 123], + ) + def test_init_rejects_invalid_metric(self, invalid_metric): + """Test initialization rejects metric that is not an instance of PlagiarismMetric.""" + with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"): + PlagiarismScorer(reference_text="Valid reference text", metric=invalid_metric) + + @pytest.mark.parametrize("invalid_n", [0, -1, 1.5, False, True, "3", None]) + def test_plagiarism_score_rejects_invalid_n(self, invalid_n): + """Test _plagiarism_score rejects invalid n.""" + scorer = PlagiarismScorer(reference_text="Valid reference text") + with pytest.raises(ValueError, match=r"n must be an integer >= 1"): + scorer._plagiarism_score(response="test", reference="test", n=invalid_n) + + @pytest.mark.parametrize("invalid_metric", ["lcs", "levenshtein", None, 123]) + def test_plagiarism_score_rejects_invalid_metric(self, invalid_metric): + """Test _plagiarism_score rejects invalid metric.""" + scorer = PlagiarismScorer(reference_text="Valid reference text") + with pytest.raises(ValueError, match="metric must be 'lcs', 'levenshtein', or 'jaccard'"): + scorer._plagiarism_score(response="test", reference="test", metric=invalid_metric) + async def test_score_async_lcs_metric(self): """Test scoring with LCS metric.""" reference_text = "The quick brown fox jumps over the lazy dog" From e7289fa251f22941c5600869a39f2bf844b2353e Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 9 Oct 2026 13:18:46 -0700 Subject: [PATCH 2/2] FIX Reject foreign metrics before plagiarism scoring shortcuts Validate the helper metric with the same enum contract as construction. Cover foreign enums and invalid inputs across empty, identical, contained, and unrelated text before returning a score. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/score/float_scale/plagiarism_scorer.py | 13 +++++-- tests/unit/score/test_plagiarism_scorer.py | 36 ++++++++++++-------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index 435ad76541..282f972402 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -166,10 +166,19 @@ def _plagiarism_score( metric: PlagiarismMetric = PlagiarismMetric.LCS, n: int = 5, ) -> float: + """ + Compute word-level similarity after validating the metric and n-gram size. + + Returns: + float: The normalized similarity score between 0 and 1. + + Raises: + ValueError: If ``metric`` is not a PlagiarismMetric or ``n`` is not an integer >= 1. + """ if not isinstance(n, int) or isinstance(n, bool) or n < 1: raise ValueError(f"n must be an integer >= 1, got {n!r}.") - if not hasattr(metric, "value"): - raise ValueError("metric must be 'lcs', 'levenshtein', or 'jaccard'") + if not isinstance(metric, PlagiarismMetric): + raise ValueError(f"metric must be an instance of PlagiarismMetric, got {metric!r}.") tokens_response = self._tokenize(response) tokens_reference = self._tokenize(reference) diff --git a/tests/unit/score/test_plagiarism_scorer.py b/tests/unit/score/test_plagiarism_scorer.py index 0d8fd6fab9..8963ea7728 100644 --- a/tests/unit/score/test_plagiarism_scorer.py +++ b/tests/unit/score/test_plagiarism_scorer.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from enum import Enum from unittest.mock import patch import pytest @@ -11,6 +12,13 @@ from pyrit.score import MessageScorable, PlagiarismMetric, PlagiarismScorer +class _OtherMetric(Enum): + LCS = "lcs" + LEVENSHTEIN = "levenshtein" + JACCARD = "jaccard" + INVALID = "invalid" + + @pytest.mark.usefixtures("patch_central_database") class TestPlagiarismScorer: """Test cases for the PlagiarismScorer class.""" @@ -71,7 +79,7 @@ def test_init_accepts_valid_boundary_n(self, valid_n): @pytest.mark.parametrize( "invalid_metric", - ["lcs", "levenshtein", "jaccard", "invalid", None, 123], + ["lcs", "levenshtein", "jaccard", "invalid", None, 123, *_OtherMetric], ) def test_init_rejects_invalid_metric(self, invalid_metric): """Test initialization rejects metric that is not an instance of PlagiarismMetric.""" @@ -85,12 +93,18 @@ def test_plagiarism_score_rejects_invalid_n(self, invalid_n): with pytest.raises(ValueError, match=r"n must be an integer >= 1"): scorer._plagiarism_score(response="test", reference="test", n=invalid_n) - @pytest.mark.parametrize("invalid_metric", ["lcs", "levenshtein", None, 123]) - def test_plagiarism_score_rejects_invalid_metric(self, invalid_metric): + @pytest.mark.parametrize("invalid_metric", ["lcs", "levenshtein", "jaccard", "invalid", None, 123, *_OtherMetric]) + @pytest.mark.parametrize( + ("response", "reference"), + [("test", "test"), ("", "test"), ("test", ""), ("different", "test"), ("prefix test suffix", "test")], + ) + def test_plagiarism_score_rejects_invalid_metric( + self, *, invalid_metric: object, response: str, reference: str + ) -> None: """Test _plagiarism_score rejects invalid metric.""" scorer = PlagiarismScorer(reference_text="Valid reference text") - with pytest.raises(ValueError, match="metric must be 'lcs', 'levenshtein', or 'jaccard'"): - scorer._plagiarism_score(response="test", reference="test", metric=invalid_metric) + with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"): + scorer._plagiarism_score(response=response, reference=reference, metric=invalid_metric) async def test_score_async_lcs_metric(self): """Test scoring with LCS metric.""" @@ -383,15 +397,9 @@ def test_plagiarism_score_empty_texts(self, scorer): assert score == 0.0 def test_plagiarism_score_invalid_metric(self, scorer): - """Test plagiarism score with mock invalid metric raises ValueError.""" - from unittest.mock import MagicMock - - # Create a mock metric that has an invalid value - mock_metric = MagicMock() - mock_metric.value = "invalid" - - with pytest.raises(ValueError, match="metric must be 'lcs', 'levenshtein', or 'jaccard'"): - scorer._plagiarism_score("hello", "world", metric=mock_metric) + """Test plagiarism score rejects an unsupported enum.""" + with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"): + scorer._plagiarism_score("hello", "world", metric=_OtherMetric.INVALID) def test_plagiarism_score_case_insensitive(self, scorer): """Test that plagiarism score is case insensitive."""