diff --git a/pyrit/scenario/core/attack_technique_factory.py b/pyrit/scenario/core/attack_technique_factory.py index cb20520bd9..e87c0befde 100644 --- a/pyrit/scenario/core/attack_technique_factory.py +++ b/pyrit/scenario/core/attack_technique_factory.py @@ -733,8 +733,8 @@ def create( into the factory or supplied via ``attack_converter_config_override``). Unlike ``attack_converter_config_override`` these are additive and never - replace the existing converters. Only forwarded if the attack - class constructor accepts ``attack_converter_config``. + replace the existing converters. Requires the attack class + constructor to accept ``attack_converter_config``. Returns: A fresh AttackTechnique with a newly-constructed attack technique. @@ -743,10 +743,19 @@ class constructor accepts ``attack_converter_config``. ValueError: If a create-time adversarial chat is supplied while the factory already baked one, if ``scorer_override_policy`` is RAISE and the scenario scorer is incompatible with the attack's type annotation, - or if ``use_score_as_feedback`` is set but no scoring config reaches the attack. + or if ``use_score_as_feedback`` is set but no scoring config reaches the attack, + or if ``extra_request_converters`` is non-empty but the attack class doesn't + accept ``attack_converter_config``. """ create_time_target: PromptTarget | None = adversarial_chat + if extra_request_converters and "attack_converter_config" not in self._compatibility_helper.accepted_params: + # These come from the caller's technique_converters, so dropping them would run a different setup. + raise ValueError( + f"Factory '{self._name}': {self._attack_class.__name__} does not accept 'attack_converter_config', " + f"so the extra request converters can't be applied." + ) + if create_time_target is not None and self._adversarial_chat is not None: raise ValueError( f"Factory '{self._name}': an adversarial chat is already baked into this technique, so " diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index da6aa5124e..20648dbac5 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -141,6 +141,10 @@ class Scenario(ABC): #: an unavailable verdict is an expected result rather than a scenario error. RAISE_IF_DEFAULT_SCORER_BLOCKS: ClassVar[bool] = True + #: Whether the scenario applies ``technique_converters``. Scenarios that don't set this to + #: False so the parameter isn't declared, and passing it fails instead of being ignored. + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = True + def __init_subclass__(cls, **kwargs: Any) -> None: """ Enforce the keyword-only constructor contract on subclasses. @@ -431,7 +435,10 @@ def supported_parameters(cls) -> list[Parameter]: Returns: list[Parameter]: Declared parameters (default: common run inputs + additional). """ - return cls._common_scenario_parameters() + cls.additional_parameters() + parameters = cls._common_scenario_parameters() + cls.additional_parameters() + if not cls.SUPPORTS_TECHNIQUE_CONVERTERS: + parameters = [parameter for parameter in parameters if parameter.name != "technique_converters"] + return parameters def _get_default_objective_scorer(self) -> TrueFalseScorer: # Deferred import to avoid circular dependency. diff --git a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py index 08a57158ca..eeb30ab0fa 100644 --- a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py +++ b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py @@ -60,6 +60,7 @@ class AdaptiveScenario(Scenario): """ VERSION: ClassVar[int] + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False @classmethod @abstractmethod diff --git a/pyrit/scenario/scenarios/airt/psychosocial.py b/pyrit/scenario/scenarios/airt/psychosocial.py index 37e68e6781..1388ca73d7 100644 --- a/pyrit/scenario/scenarios/airt/psychosocial.py +++ b/pyrit/scenario/scenarios/airt/psychosocial.py @@ -7,7 +7,7 @@ import logging import pathlib from dataclasses import dataclass -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, ClassVar, cast from pyrit.common import apply_defaults from pyrit.common.path import DATASETS_PATH @@ -369,6 +369,7 @@ class Psychosocial(Scenario): """ VERSION: int = 4 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False @classmethod def additional_parameters(cls) -> list[Parameter]: diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index 686e277b24..12c02b4f90 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -4,7 +4,7 @@ import asyncio import logging from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from pyrit.common import apply_defaults from pyrit.common.path import EXECUTOR_RED_TEAM_PATH, EXECUTOR_SIMULATED_TARGET_PATH, SCORER_SEED_PROMPT_PATH @@ -90,6 +90,7 @@ class Scam(Scenario): """ VERSION: int = 2 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False @classmethod def _get_additional_scoring_questions(cls) -> list[Path]: diff --git a/pyrit/scenario/scenarios/benchmark/adversarial.py b/pyrit/scenario/scenarios/benchmark/adversarial.py index 749c2e5ae1..ba7033ee19 100644 --- a/pyrit/scenario/scenarios/benchmark/adversarial.py +++ b/pyrit/scenario/scenarios/benchmark/adversarial.py @@ -528,6 +528,7 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list adversarial_targets=resolved_targets, display_group_fn=lambda combo: combo.target_name or "", include_baseline=context.include_baseline, + technique_converters=self._technique_converters, ) if not self._is_cache_reuse_enabled() or self._scenario_result_id: return atomic_attacks diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index 1b46e26183..b432f54b82 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -253,6 +253,7 @@ class RedTeamAgent(Scenario): """ VERSION: int = 1 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False _DEFAULT_ATTACK_SPECIFICATION: ClassVar[_AttackSpecification] = _AttackSpecification(PromptSendingAttack) _ATTACK_SPECIFICATIONS: ClassVar[Mapping[FoundryTechnique, _AttackSpecification]] = MappingProxyType( { diff --git a/pyrit/scenario/scenarios/garak/audio_achilles_heel.py b/pyrit/scenario/scenarios/garak/audio_achilles_heel.py index b332b7a468..e64cbcc27b 100644 --- a/pyrit/scenario/scenarios/garak/audio_achilles_heel.py +++ b/pyrit/scenario/scenarios/garak/audio_achilles_heel.py @@ -167,6 +167,7 @@ class AudioAchillesHeel(Scenario): """ VERSION: int = 1 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False # Audio payload dominated: a text-only baseline of the derived objective drops the audio # entirely and is a weak comparison point (Garak has no text-only variant). Baseline stays diff --git a/pyrit/scenario/scenarios/garak/doctor.py b/pyrit/scenario/scenarios/garak/doctor.py index 1aed3279a9..82239b63e1 100644 --- a/pyrit/scenario/scenarios/garak/doctor.py +++ b/pyrit/scenario/scenarios/garak/doctor.py @@ -172,4 +172,5 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list technique_factories=technique_factories, dataset_groups=context.seed_groups_by_dataset, include_baseline=context.include_baseline, + technique_converters=self._technique_converters, ) diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index e1a95d9108..e0dbf1854c 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -4,6 +4,7 @@ import logging from collections.abc import Sequence +from typing import ClassVar from pyrit.common import apply_defaults from pyrit.converter import ( @@ -163,6 +164,7 @@ class Encoding(Scenario): """ VERSION: int = 2 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False @apply_defaults def __init__( diff --git a/pyrit/scenario/scenarios/garak/package_hallucination.py b/pyrit/scenario/scenarios/garak/package_hallucination.py index b4d9b0c97f..d9f9434b94 100644 --- a/pyrit/scenario/scenarios/garak/package_hallucination.py +++ b/pyrit/scenario/scenarios/garak/package_hallucination.py @@ -153,6 +153,7 @@ class PackageHallucination(Scenario): """ VERSION: int = 3 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False # The plain code request is not an adversarial baseline to compare against, so no baseline. BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden diff --git a/pyrit/scenario/scenarios/garak/system_prompt_extraction.py b/pyrit/scenario/scenarios/garak/system_prompt_extraction.py index cb8ff5e4c4..6062e8f208 100644 --- a/pyrit/scenario/scenarios/garak/system_prompt_extraction.py +++ b/pyrit/scenario/scenarios/garak/system_prompt_extraction.py @@ -97,6 +97,7 @@ class SystemPromptExtraction(Scenario): """ VERSION: int = 1 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False # Template-dominated like the Doctor/Jailbreak scenarios: the bare system prompt with no # extraction request is a weak comparison point, so baseline is off by default. diff --git a/pyrit/scenario/scenarios/garak/web_injection.py b/pyrit/scenario/scenarios/garak/web_injection.py index 11b064b0f8..4c10ce2e67 100644 --- a/pyrit/scenario/scenarios/garak/web_injection.py +++ b/pyrit/scenario/scenarios/garak/web_injection.py @@ -103,6 +103,7 @@ class WebInjection(Scenario): """ VERSION: int = 2 + SUPPORTS_TECHNIQUE_CONVERTERS: ClassVar[bool] = False USES_DATASET_SIZE_LIMIT: ClassVar[bool] = False BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Enabled diff --git a/tests/unit/scenario/benchmark/test_adversarial.py b/tests/unit/scenario/benchmark/test_adversarial.py index 5239843afb..b925f64438 100644 --- a/tests/unit/scenario/benchmark/test_adversarial.py +++ b/tests/unit/scenario/benchmark/test_adversarial.py @@ -931,6 +931,18 @@ async def test_cross_product_count_matches_n_techniques_m_targets_d_datasets(sel result = await _build_atomic_attacks(bench) assert len(result) == 2 + async def test_technique_converters_are_passed_to_the_builder(self): + bench = self._make_bench_with_targets(target_names=["adv_a"]) + converters = {"red_teaming": [MagicMock()]} + bench._technique_converters = converters + + with patch( + "pyrit.scenario.scenarios.benchmark.adversarial.MatrixAtomicAttackBuilder.build", return_value=[] + ) as build: + await _build_atomic_attacks(bench) + + assert build.call_args.kwargs["technique_converters"] is converters + async def test_atomic_attack_name_format_is_technique__target_dataset(self): """Name format: ``{technique}__{target}_{dataset}`` (preserves VERSION=2 cache key shape).""" bench = self._make_bench_with_targets(target_names=["adv_a"]) diff --git a/tests/unit/scenario/core/test_attack_technique_factory.py b/tests/unit/scenario/core/test_attack_technique_factory.py index e1d13e17f7..0abf15f492 100644 --- a/tests/unit/scenario/core/test_attack_technique_factory.py +++ b/tests/unit/scenario/core/test_attack_technique_factory.py @@ -494,8 +494,8 @@ def test_create_appends_extra_request_converters_on_top_of_baked(self): assert cfg.request_converters == baked_request + extra assert cfg.response_converters == baked_response - def test_create_extra_request_converters_skipped_when_unsupported(self): - """Attacks that don't accept ``attack_converter_config`` silently ignore extras.""" + def test_create_extra_request_converters_raise_when_unsupported(self): + """Attacks that don't accept ``attack_converter_config`` reject extras instead of dropping them.""" class _NoConverterAttack: def __init__(self, *, objective_target, attack_scoring_config=None): @@ -508,13 +508,12 @@ def get_identifier(self): target = MagicMock(spec=PromptTarget) extra = ConverterConfiguration.from_converters(converters=[Base64Converter()]) - technique = factory.create( - objective_target=target, - attack_scoring_config=self._scoring(), - extra_request_converters=extra, - ) - - assert isinstance(technique, AttackTechnique) + with pytest.raises(ValueError, match="does not accept 'attack_converter_config'"): + factory.create( + objective_target=target, + attack_scoring_config=self._scoring(), + extra_request_converters=extra, + ) def test_create_with_deferred_forward_ref_scoring_config_policy_raise(self): """Forward-referenced scoring config defined after factory init resolves and raises on incompatible type.""" diff --git a/tests/unit/scenario/core/test_attack_technique_factory_converter_composition.py b/tests/unit/scenario/core/test_attack_technique_factory_converter_composition.py index c24b6286bc..a6e7e06a3e 100644 --- a/tests/unit/scenario/core/test_attack_technique_factory_converter_composition.py +++ b/tests/unit/scenario/core/test_attack_technique_factory_converter_composition.py @@ -5,12 +5,18 @@ from unittest.mock import MagicMock +import pytest +from unit.mocks import MockPromptTarget + from pyrit.converter import Base64Converter, ROT13Converter +from pyrit.executor.attack import PromptSendingAttack from pyrit.executor.attack.core.attack_config import AttackConverterConfig, AttackScoringConfig -from pyrit.models import ComponentIdentifier +from pyrit.models import AttackSeedGroup, ComponentIdentifier, SeedObjective from pyrit.prompt_normalizer import ConverterConfiguration from pyrit.prompt_target import PromptTarget from pyrit.scenario.core.attack_technique_factory import AttackTechniqueFactory +from pyrit.scenario.core.matrix_atomic_attack_builder import MatrixAtomicAttackBuilder +from pyrit.score import TrueFalseScorer class _ConverterAttack: @@ -177,3 +183,50 @@ def test_create_delegates_converter_composition_to_private_helper(monkeypatch): helper.assert_called_once() assert technique.attack.attack_converter_config is composed + + +class _NoConverterAttack: + def __init__(self, *, objective_target: PromptTarget, attack_scoring_config: AttackScoringConfig | None = None): + self.objective_target = objective_target + self.attack_scoring_config = attack_scoring_config + + def get_identifier(self) -> ComponentIdentifier: + return ComponentIdentifier(class_name="_NoConverterAttack", class_module="test") + + +def test_extras_raise_when_the_attack_cannot_take_converters(): + factory = AttackTechniqueFactory(name="custom", attack_class=_NoConverterAttack) + extras = ConverterConfiguration.from_converters(converters=[Base64Converter()]) + + with pytest.raises(ValueError, match="does not accept 'attack_converter_config'"): + _create(factory, extra_request_converters=extras) + + +def test_no_extras_still_work_when_the_attack_cannot_take_converters(): + factory = AttackTechniqueFactory(name="custom", attack_class=_NoConverterAttack) + + assert isinstance(_create(factory).attack, _NoConverterAttack) + assert isinstance(_create(factory, extra_request_converters=[]).attack, _NoConverterAttack) + + +class _NoConverterPromptSendingAttack(PromptSendingAttack): + """A real registered-style attack whose constructor has no attack_converter_config.""" + + def __init__(self, *, objective_target: PromptTarget, attack_scoring_config: AttackScoringConfig | None = None): + super().__init__(objective_target=objective_target, attack_scoring_config=attack_scoring_config) + + +@pytest.mark.usefixtures("patch_central_database") +def test_matrix_builder_rejects_technique_converters_a_real_attack_cannot_apply(): + target = MockPromptTarget() + scorer = MagicMock(spec=TrueFalseScorer) + scorer.get_identifier.return_value = ComponentIdentifier(class_name="Scorer", class_module="test") + builder = MatrixAtomicAttackBuilder(objective_target=target, objective_scorer=scorer) + factory = AttackTechniqueFactory(name="custom", attack_class=_NoConverterPromptSendingAttack) + + with pytest.raises(ValueError, match="does not accept 'attack_converter_config'"): + builder.build( + technique_factories={"custom": factory}, + dataset_groups={"ds": [AttackSeedGroup(seeds=[SeedObjective(value="objective")])]}, + technique_converters={"custom": [Base64Converter()]}, + ) diff --git a/tests/unit/scenario/garak/test_doctor.py b/tests/unit/scenario/garak/test_doctor.py index 5d4bf552ea..7d459773bf 100644 --- a/tests/unit/scenario/garak/test_doctor.py +++ b/tests/unit/scenario/garak/test_doctor.py @@ -7,7 +7,7 @@ import pytest -from pyrit.converter import LeetspeakConverter, PolicyPuppetryConverter +from pyrit.converter import Base64Converter, LeetspeakConverter, PolicyPuppetryConverter from pyrit.executor.attack import PromptSendingAttack from pyrit.models import ComponentIdentifier, SeedGroup, SeedObjective from pyrit.prompt_target import PromptTarget @@ -190,6 +190,27 @@ async def test_atomic_attacks_one_per_technique( assert any(n.startswith("policy_puppetry") and "leet" not in n for n in names) assert all(isinstance(a.attack_technique.attack, PromptSendingAttack) for a in atomic_attacks) + async def test_technique_converters_are_appended_to_their_technique( + self, mock_objective_target, mock_objective_scorer, doctor_dataset_config + ): + scenario = Doctor(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args( + args={ + "objective_target": mock_objective_target, + "dataset_config": doctor_dataset_config, + "technique_converters": {"policy_puppetry": [Base64Converter()]}, + } + ) + await scenario.initialize_async() + + converters_by_attack = { + attack.atomic_attack_name: _flatten_converters(attack.attack_technique.attack) + for attack in scenario._atomic_attacks + } + for name, converters in converters_by_attack.items(): + has_base64 = any(isinstance(c, Base64Converter) for c in converters) + assert has_base64 == ("leet" not in name) + @pytest.mark.usefixtures("patch_central_database") class TestDoctorTechniqueTags: diff --git a/tests/unit/scenario/test_technique_converters_support.py b/tests/unit/scenario/test_technique_converters_support.py new file mode 100644 index 0000000000..16fc32dec7 --- /dev/null +++ b/tests/unit/scenario/test_technique_converters_support.py @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Scenarios either apply ``technique_converters`` or don't accept it, so it's never silently ignored.""" + +from unittest.mock import MagicMock + +import pytest + +from pyrit.converter import Base64Converter +from pyrit.models import ComponentIdentifier +from pyrit.scenario.scenarios.adaptive.adaptive_scenario import AdaptiveScenario +from pyrit.scenario.scenarios.adaptive.text_adaptive import TextAdaptive +from pyrit.scenario.scenarios.airt.cyber import Cyber +from pyrit.scenario.scenarios.airt.psychosocial import Psychosocial +from pyrit.scenario.scenarios.airt.scam import Scam +from pyrit.scenario.scenarios.benchmark.adversarial import AdversarialBenchmark +from pyrit.scenario.scenarios.foundry.red_team_agent import RedTeamAgent +from pyrit.scenario.scenarios.garak.audio_achilles_heel import AudioAchillesHeel +from pyrit.scenario.scenarios.garak.doctor import Doctor +from pyrit.scenario.scenarios.garak.encoding import Encoding +from pyrit.scenario.scenarios.garak.package_hallucination import PackageHallucination +from pyrit.scenario.scenarios.garak.system_prompt_extraction import SystemPromptExtraction +from pyrit.scenario.scenarios.garak.web_injection import WebInjection +from pyrit.score import TrueFalseScorer + +_IGNORING_SCENARIOS = [ + AdaptiveScenario, + TextAdaptive, + AudioAchillesHeel, + Encoding, + PackageHallucination, + Psychosocial, + RedTeamAgent, + Scam, + SystemPromptExtraction, + WebInjection, +] + + +def _declares_technique_converters(scenario_class: type) -> bool: + return any(parameter.name == "technique_converters" for parameter in scenario_class.supported_parameters()) + + +@pytest.mark.parametrize("scenario_class", _IGNORING_SCENARIOS, ids=lambda cls: cls.__name__) +def test_scenarios_that_dont_apply_technique_converters_dont_declare_them(scenario_class: type) -> None: + assert not _declares_technique_converters(scenario_class) + + +@pytest.mark.parametrize("scenario_class", [AdversarialBenchmark, Cyber, Doctor], ids=lambda cls: cls.__name__) +def test_scenarios_that_apply_technique_converters_declare_them(scenario_class: type) -> None: + assert _declares_technique_converters(scenario_class) + + +@pytest.mark.usefixtures("patch_central_database") +def test_passing_technique_converters_to_a_scenario_that_ignores_them_raises() -> None: + scorer = MagicMock(spec=TrueFalseScorer) + scorer.get_identifier.return_value = ComponentIdentifier(class_name="Scorer", class_module="test") + scenario = Encoding(objective_scorer=scorer) + + with pytest.raises(ValueError, match="unknown parameter.*technique_converters"): + scenario.set_params_from_args(args={"technique_converters": {"base64": [Base64Converter()]}})