FIX: validate n-gram size and reference text in PlagiarismScorer

pyrit/score/float_scale/plagiarism_scorer.py

@@ -48,13 +48,27 @@ def __init__(
             metric (PlagiarismMetric): The plagiarism detection metric to use. Defaults to PlagiarismMetric.LCS.
             n (int): The n-gram size for n-gram similarity. Defaults to 5.
             validator (ScorerPromptValidator | None): Custom validator for the scorer. Defaults to None.
+
+        Raises:
+            ValueError: If ``reference_text`` is not a non-empty string or contains no word tokens,
+                if ``metric`` is not an instance of PlagiarismMetric, or if ``n`` is not an integer >= 1.
         """
-        super().__init__(validator=validator or self._DEFAULT_VALIDATOR)
+        if not isinstance(reference_text, str) or not reference_text.strip():
+            raise ValueError("reference_text must be a non-empty string.")
+        if not isinstance(metric, PlagiarismMetric):
+            raise ValueError(f"metric must be an instance of PlagiarismMetric, got {metric!r}.")
+        if not isinstance(n, int) or isinstance(n, bool) or n < 1:
+            raise ValueError(f"n must be an integer >= 1, got {n!r}.")
 
         self.reference_text = reference_text
         self.metric = metric
         self.n = n
 
+        if not self._tokenize(reference_text):
+            raise ValueError("reference_text must contain at least one word token.")
+
+        super().__init__(validator=validator or self._DEFAULT_VALIDATOR)
+
     def _build_identifier(self) -> ComponentIdentifier:
         """
         Build the identifier for this scorer.
@@ -152,6 +166,20 @@ def _plagiarism_score(
         metric: PlagiarismMetric = PlagiarismMetric.LCS,
         n: int = 5,
     ) -> float:
+        """
+        Compute word-level similarity after validating the metric and n-gram size.
+
+        Returns:
+            float: The normalized similarity score between 0 and 1.
+
+        Raises:
+            ValueError: If ``metric`` is not a PlagiarismMetric or ``n`` is not an integer >= 1.
+        """
+        if not isinstance(n, int) or isinstance(n, bool) or n < 1:
+            raise ValueError(f"n must be an integer >= 1, got {n!r}.")
+        if not isinstance(metric, PlagiarismMetric):
+            raise ValueError(f"metric must be an instance of PlagiarismMetric, got {metric!r}.")
+
         tokens_response = self._tokenize(response)
         tokens_reference = self._tokenize(reference)
         response_len = len(tokens_response)

tests/unit/score/test_plagiarism_scorer.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+from enum import Enum
 from unittest.mock import patch
 
 import pytest
@@ -11,6 +12,13 @@
 from pyrit.score import MessageScorable, PlagiarismMetric, PlagiarismScorer
 
 
+class _OtherMetric(Enum):
+    LCS = "lcs"
+    LEVENSHTEIN = "levenshtein"
+    JACCARD = "jaccard"
+    INVALID = "invalid"
+
+
 @pytest.mark.usefixtures("patch_central_database")
 class TestPlagiarismScorer:
     """Test cases for the PlagiarismScorer class."""
@@ -36,6 +44,68 @@ def test_init_with_custom_parameters(self):
         assert scorer.metric == metric
         assert scorer.n == n
 
+    @pytest.mark.parametrize(
+        "invalid_reference",
+        ["", "   ", "\t\n  ", None, 123, [], {}],
+    )
+    def test_init_rejects_empty_or_non_string_reference_text(self, invalid_reference):
+        """Test initialization rejects empty, whitespace-only, or non-string reference text."""
+        with pytest.raises(ValueError, match="reference_text must be a non-empty string"):
+            PlagiarismScorer(reference_text=invalid_reference)
+
+    @pytest.mark.parametrize(
+        "no_token_reference",
+        ["!!!", "???", "---", "... ,,, ;;;", "   !@#$%^&*()   "],
+    )
+    def test_init_rejects_reference_text_without_tokens(self, no_token_reference):
+        """Test initialization rejects reference text containing no word tokens."""
+        with pytest.raises(ValueError, match="reference_text must contain at least one word token"):
+            PlagiarismScorer(reference_text=no_token_reference)
+
+    @pytest.mark.parametrize(
+        "invalid_n",
+        [0, -1, -5, 1.5, False, True, "3", None, [3]],
+    )
+    def test_init_rejects_invalid_n(self, invalid_n):
+        """Test initialization rejects n that is not an integer >= 1 or is a boolean."""
+        with pytest.raises(ValueError, match=r"n must be an integer >= 1"):
+            PlagiarismScorer(reference_text="Valid reference text", n=invalid_n)
+
+    @pytest.mark.parametrize("valid_n", [1, 2, 5, 10])
+    def test_init_accepts_valid_boundary_n(self, valid_n):
+        """Test initialization accepts positive integer n-gram sizes."""
+        scorer = PlagiarismScorer(reference_text="Valid reference text", n=valid_n)
+        assert scorer.n == valid_n
+
+    @pytest.mark.parametrize(
+        "invalid_metric",
+        ["lcs", "levenshtein", "jaccard", "invalid", None, 123, *_OtherMetric],
+    )
+    def test_init_rejects_invalid_metric(self, invalid_metric):
+        """Test initialization rejects metric that is not an instance of PlagiarismMetric."""
+        with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"):
+            PlagiarismScorer(reference_text="Valid reference text", metric=invalid_metric)
+
+    @pytest.mark.parametrize("invalid_n", [0, -1, 1.5, False, True, "3", None])
+    def test_plagiarism_score_rejects_invalid_n(self, invalid_n):
+        """Test _plagiarism_score rejects invalid n."""
+        scorer = PlagiarismScorer(reference_text="Valid reference text")
+        with pytest.raises(ValueError, match=r"n must be an integer >= 1"):
+            scorer._plagiarism_score(response="test", reference="test", n=invalid_n)
+
+    @pytest.mark.parametrize("invalid_metric", ["lcs", "levenshtein", "jaccard", "invalid", None, 123, *_OtherMetric])
+    @pytest.mark.parametrize(
+        ("response", "reference"),
+        [("test", "test"), ("", "test"), ("test", ""), ("different", "test"), ("prefix test suffix", "test")],
+    )
+    def test_plagiarism_score_rejects_invalid_metric(
+        self, *, invalid_metric: object, response: str, reference: str
+    ) -> None:
+        """Test _plagiarism_score rejects invalid metric."""
+        scorer = PlagiarismScorer(reference_text="Valid reference text")
+        with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"):
+            scorer._plagiarism_score(response=response, reference=reference, metric=invalid_metric)
+
     async def test_score_async_lcs_metric(self):
         """Test scoring with LCS metric."""
         reference_text = "The quick brown fox jumps over the lazy dog"
@@ -327,15 +397,9 @@ def test_plagiarism_score_empty_texts(self, scorer):
         assert score == 0.0
 
     def test_plagiarism_score_invalid_metric(self, scorer):
-        """Test plagiarism score with mock invalid metric raises ValueError."""
-        from unittest.mock import MagicMock
-
-        # Create a mock metric that has an invalid value
-        mock_metric = MagicMock()
-        mock_metric.value = "invalid"
-
-        with pytest.raises(ValueError, match="metric must be 'lcs', 'levenshtein', or 'jaccard'"):
-            scorer._plagiarism_score("hello", "world", metric=mock_metric)
+        """Test plagiarism score rejects an unsupported enum."""
+        with pytest.raises(ValueError, match="metric must be an instance of PlagiarismMetric"):
+            scorer._plagiarism_score("hello", "world", metric=_OtherMetric.INVALID)
 
     def test_plagiarism_score_case_insensitive(self, scorer):
         """Test that plagiarism score is case insensitive."""