FIX Stop silently ignoring technique_converters

pyrit/scenario/core/scenario.py

@@ -141,6 +141,10 @@ class Scenario(ABC):
     #: an unavailable verdict is an expected result rather than a scenario error.
     RAISE_IF_DEFAULT_SCORER_BLOCKS: ClassVar[bool] = True
 
+    #: Whether the scenario applies ``technique_converters``. Scenarios that don't set this to
+    #: False so the parameter isn't declared, and passing it fails instead of being ignored.
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = True
+
     def __init_subclass__(cls, **kwargs: Any) -> None:
         """
         Enforce the keyword-only constructor contract on subclasses.
@@ -431,7 +435,10 @@ def supported_parameters(cls) -> list[Parameter]:
         Returns:
             list[Parameter]: Declared parameters (default: common run inputs + additional).
         """
-        return cls._common_scenario_parameters() + cls.additional_parameters()
+        parameters = cls._common_scenario_parameters() + cls.additional_parameters()
+        if not cls.SUPPORTS_TECHNIQUE_CONVERTERS:
+            parameters = [parameter for parameter in parameters if parameter.name != "technique_converters"]
+        return parameters
 
     def _get_default_objective_scorer(self) -> TrueFalseScorer:
         # Deferred import to avoid circular dependency.

pyrit/scenario/scenarios/adaptive/adaptive_scenario.py

@@ -60,6 +60,7 @@ class AdaptiveScenario(Scenario):
     """
 
     VERSION: ClassVar[int]
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     @classmethod
     @abstractmethod

pyrit/scenario/scenarios/airt/psychosocial.py

@@ -7,7 +7,7 @@
 import logging
 import pathlib
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, ClassVar, cast
 
 from pyrit.common import apply_defaults
 from pyrit.common.path import DATASETS_PATH
@@ -369,6 +369,7 @@ class Psychosocial(Scenario):
     """
 
     VERSION: int = 4
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     @classmethod
     def additional_parameters(cls) -> list[Parameter]:

pyrit/scenario/scenarios/airt/scam.py

@@ -4,7 +4,7 @@
 import asyncio
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from pyrit.common import apply_defaults
 from pyrit.common.path import EXECUTOR_RED_TEAM_PATH, EXECUTOR_SIMULATED_TARGET_PATH, SCORER_SEED_PROMPT_PATH
@@ -90,6 +90,7 @@ class Scam(Scenario):
     """
 
     VERSION: int = 2
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     @classmethod
     def _get_additional_scoring_questions(cls) -> list[Path]:

pyrit/scenario/scenarios/benchmark/adversarial.py

@@ -528,6 +528,7 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list
             adversarial_targets=resolved_targets,
             display_group_fn=lambda combo: combo.target_name or "",
             include_baseline=context.include_baseline,
+            technique_converters=self._technique_converters,
         )
         if not self._is_cache_reuse_enabled() or self._scenario_result_id:
             return atomic_attacks

pyrit/scenario/scenarios/foundry/red_team_agent.py

@@ -253,6 +253,7 @@ class RedTeamAgent(Scenario):
     """
 
     VERSION: int = 1
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
     _DEFAULT_ATTACK_SPECIFICATION: ClassVar[_AttackSpecification] = _AttackSpecification(PromptSendingAttack)
     _ATTACK_SPECIFICATIONS: ClassVar[Mapping[FoundryTechnique, _AttackSpecification]] = MappingProxyType(
         {

pyrit/scenario/scenarios/garak/audio_achilles_heel.py

@@ -167,6 +167,7 @@ class AudioAchillesHeel(Scenario):
     """
 
     VERSION: int = 1
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     # Audio payload dominated: a text-only baseline of the derived objective drops the audio
     # entirely and is a weak comparison point (Garak has no text-only variant). Baseline stays

pyrit/scenario/scenarios/garak/doctor.py

@@ -172,4 +172,5 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list
             technique_factories=technique_factories,
             dataset_groups=context.seed_groups_by_dataset,
             include_baseline=context.include_baseline,
+            technique_converters=self._technique_converters,
         )

pyrit/scenario/scenarios/garak/encoding.py

@@ -4,6 +4,7 @@
 
 import logging
 from collections.abc import Sequence
+from typing import ClassVar
 
 from pyrit.common import apply_defaults
 from pyrit.converter import (
@@ -163,6 +164,7 @@ class Encoding(Scenario):
     """
 
     VERSION: int = 2
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     @apply_defaults
     def __init__(

pyrit/scenario/scenarios/garak/package_hallucination.py

@@ -153,6 +153,7 @@ class PackageHallucination(Scenario):
     """
 
     VERSION: int = 3
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     # The plain code request is not an adversarial baseline to compare against, so no baseline.
     BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden

pyrit/scenario/scenarios/garak/system_prompt_extraction.py

@@ -97,6 +97,7 @@ class SystemPromptExtraction(Scenario):
     """
 
     VERSION: int = 1
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
 
     # Template-dominated like the Doctor/Jailbreak scenarios: the bare system prompt with no
     # extraction request is a weak comparison point, so baseline is off by default.

pyrit/scenario/scenarios/garak/web_injection.py

@@ -99,6 +99,7 @@ class WebInjection(Scenario):
     """
 
     VERSION: int = 2
+    SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False
     USES_DATASET_SIZE_LIMIT: ClassVar[bool] = False
     BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled
 

tests/unit/scenario/benchmark/test_adversarial.py

@@ -931,6 +931,18 @@ async def test_cross_product_count_matches_n_techniques_m_targets_d_datasets(sel
         result = await _build_atomic_attacks(bench)
         assert len(result) == 2
 
+    async def test_technique_converters_are_passed_to_the_builder(self):
+        bench = self._make_bench_with_targets(target_names=["adv_a"])
+        converters = {"red_teaming": [MagicMock()]}
+        bench._technique_converters = converters
+
+        with patch(
+            "pyrit.scenario.scenarios.benchmark.adversarial.MatrixAtomicAttackBuilder.build", return_value=[]
+        ) as build:
+            await _build_atomic_attacks(bench)
+
+        assert build.call_args.kwargs["technique_converters"] is converters
+
     async def test_atomic_attack_name_format_is_technique__target_dataset(self):
         """Name format: ``{technique}__{target}_{dataset}`` (preserves VERSION=2 cache key shape)."""
         bench = self._make_bench_with_targets(target_names=["adv_a"])

tests/unit/scenario/garak/test_doctor.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from pyrit.converter import LeetspeakConverter, PolicyPuppetryConverter
+from pyrit.converter import Base64Converter, LeetspeakConverter, PolicyPuppetryConverter
 from pyrit.executor.attack import PromptSendingAttack
 from pyrit.models import ComponentIdentifier, SeedGroup, SeedObjective
 from pyrit.prompt_target import PromptTarget
@@ -190,6 +190,27 @@ async def test_atomic_attacks_one_per_technique(
         assert any(n.startswith("policy_puppetry") and "leet" not in n for n in names)
         assert all(isinstance(a.attack_technique.attack, PromptSendingAttack) for a in atomic_attacks)
 
+    async def test_technique_converters_are_appended_to_their_technique(
+        self, mock_objective_target, mock_objective_scorer, doctor_dataset_config
+    ):
+        scenario = Doctor(objective_scorer=mock_objective_scorer)
+        scenario.set_params_from_args(
+            args={
+                "objective_target": mock_objective_target,
+                "dataset_config": doctor_dataset_config,
+                "technique_converters": {"policy_puppetry": [Base64Converter()]},
+            }
+        )
+        await scenario.initialize_async()
+
+        converters_by_attack = {
+            attack.atomic_attack_name: _flatten_converters(attack.attack_technique.attack)
+            for attack in scenario._atomic_attacks
+        }
+        for name, converters in converters_by_attack.items():
+            has_base64 = any(isinstance(c, Base64Converter) for c in converters)
+            assert has_base64 == ("leet" not in name)
+
 
 @pytest.mark.usefixtures("patch_central_database")
 class TestDoctorTechniqueTags:

tests/unit/scenario/test_technique_converters_support.py

@@ -0,0 +1,62 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Scenarios either apply ``technique_converters`` or don't accept it, so it's never silently ignored."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from pyrit.converter import Base64Converter
+from pyrit.models import ComponentIdentifier
+from pyrit.scenario.scenarios.adaptive.adaptive_scenario import AdaptiveScenario
+from pyrit.scenario.scenarios.adaptive.text_adaptive import TextAdaptive
+from pyrit.scenario.scenarios.airt.cyber import Cyber
+from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial
+from pyrit.scenario.scenarios.airt.scam import Scam
+from pyrit.scenario.scenarios.benchmark.adversarial import AdversarialBenchmark
+from pyrit.scenario.scenarios.foundry.red_team_agent import RedTeamAgent
+from pyrit.scenario.scenarios.garak.audio_achilles_heel import AudioAchillesHeel
+from pyrit.scenario.scenarios.garak.doctor import Doctor
+from pyrit.scenario.scenarios.garak.encoding import Encoding
+from pyrit.scenario.scenarios.garak.package_hallucination import PackageHallucination
+from pyrit.scenario.scenarios.garak.system_prompt_extraction import SystemPromptExtraction
+from pyrit.scenario.scenarios.garak.web_injection import WebInjection
+from pyrit.score import TrueFalseScorer
+
+_IGNORING_SCENARIOS = [
+    AdaptiveScenario,
+    TextAdaptive,
+    AudioAchillesHeel,
+    Encoding,
+    PackageHallucination,
+    Psychosocial,
+    RedTeamAgent,
+    Scam,
+    SystemPromptExtraction,
+    WebInjection,
+]
+
+
+def _declares_technique_converters(scenario_class: type) -> bool:
+    return any(parameter.name == "technique_converters" for parameter in scenario_class.supported_parameters())
+
+
+@pytest.mark.parametrize("scenario_class", _IGNORING_SCENARIOS, ids=lambda cls: cls.__name__)
+def test_scenarios_that_dont_apply_technique_converters_dont_declare_them(scenario_class: type) -> None:
+    assert not _declares_technique_converters(scenario_class)
+
+
+@pytest.mark.parametrize("scenario_class", [AdversarialBenchmark, Cyber, Doctor], ids=lambda cls: cls.__name__)
+def test_scenarios_that_apply_technique_converters_declare_them(scenario_class: type) -> None:
+    assert _declares_technique_converters(scenario_class)
+
+
+@pytest.mark.usefixtures("patch_central_database")
+def test_passing_technique_converters_to_a_scenario_that_ignores_them_raises() -> None:
+    scorer = MagicMock(spec=TrueFalseScorer)
+    scorer.get_identifier.return_value = ComponentIdentifier(class_name="Scorer", class_module="test")
+    scenario = Encoding(objective_scorer=scorer)
+
+    with pytest.raises(ValueError, match="unknown parameter.*technique_converters"):
+        scenario.set_params_from_args(args={"technique_converters": {"base64": [Base64Converter()]}})