diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 0e1cdab013..751ad55030 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -140,6 +140,12 @@ allow_custom_initializers: false # ~/.pyrit/custom_initializers. # custom_initializers_source: https://account.blob.core.windows.net/container/custom_initializers +# Optional storage for scenario presets, the saved scenario configurations the +# scanner UI launches from. This may be a local directory or an Azure Blob +# container URI with an optional blob prefix. Container URIs may include a SAS; +# otherwise DefaultAzureCredential is used. Defaults to ~/.pyrit/scenario_presets. +# scenario_presets_source: https://account.blob.core.windows.net/container/scenario_presets + # Local Backend Server # -------------------- # Client settings used by pyrit_scan when connecting to or launching a backend. diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index 20f922c9bd..da2d4b2e52 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -164,6 +164,16 @@ custom_initializers_source: https://account.blob.core.windows.net/pyrit-storage/ With this configuration, PyRIT reads and writes scripts directly under the `custom-initializers/` prefix in the `pyrit-storage` container. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/custom_initializers`. +### `scenario_presets_source` + +Stores scenario presets in a local directory or Azure Blob container. An Azure URI may include a blob-name prefix, which behaves like a folder: + +```yaml +scenario_presets_source: https://account.blob.core.windows.net/pyrit-storage/scenario-presets +``` + +Point several deployments at the same container to share presets between them. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/scenario_presets`. + ### `initialization_scripts` Local paths to custom Python scripts containing `PyRITInitializer` subclasses. Paths can be absolute or relative to the current working directory. diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index f86bddcfbf..6756d0deed 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -395,6 +395,32 @@ jest.mock("./components/Scenarios/ScenarioCatalog", () => { }; }); +jest.mock("./components/ScenarioPresets/ScenarioPresetLibrary", () => { + const MockScenarioPresetLibrary = () =>
; + MockScenarioPresetLibrary.displayName = "MockScenarioPresetLibrary"; + return { + __esModule: true, + default: MockScenarioPresetLibrary, + }; +}); + +jest.mock("./components/ScenarioPresets/ScenarioPresetEditor", () => { + const { useParams } = jest.requireActual("react-router"); + const MockScenarioPresetEditor = ({ mode }: { mode: string }) => { + const { presetName } = useParams(); + return ( +
+ {presetName ?? ""} +
+ ); + }; + MockScenarioPresetEditor.displayName = "MockScenarioPresetEditor"; + return { + __esModule: true, + default: MockScenarioPresetEditor, + }; +}); + jest.mock("./components/Scenarios/ScenarioDetail", () => { const MockScenarioDetail = ({ defaultObjectiveTarget, @@ -604,6 +630,46 @@ describe("App", () => { expect(screen.getByLabelText("Current URL")).toHaveTextContent(/^\/chat$/); }); + it("renders the preset library as a registry section", () => { + renderApp("/registry/scenario-presets"); + + expect(screen.getByTestId("main-layout")).toHaveAttribute( + "data-current-view", + "registry" + ); + expect(screen.getByTestId("scenario-preset-library")).toBeInTheDocument(); + }); + + it("redirects a bookmarked /scanner/presets to the registry section", async () => { + render( + + + + + ); + + expect(await screen.findByTestId("scenario-preset-library")).toBeInTheDocument(); + expect(screen.getByLabelText("Current URL")).toHaveTextContent( + /^\/registry\/scenario-presets$/ + ); + }); + + it("redirects a bookmarked preset editor URL, keeping the preset it named", async () => { + render( + + + + + ); + + const editor = await screen.findByTestId("scenario-preset-editor"); + expect(editor).toHaveAttribute("data-mode", "edit"); + expect(editor).toHaveTextContent("nightly_probe"); + expect(screen.getByLabelText("Current URL")).toHaveTextContent( + /^\/registry\/scenario-presets\/nightly_probe\/edit$/ + ); + }); + it("renders the converter registry from its direct URL", async () => { renderApp("/registry/converters"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e110b0e8c1..54dfe849fa 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -21,6 +21,9 @@ import ScenarioHistory from './components/History/ScenarioHistory' import ScenarioCatalog from './components/Scenarios/ScenarioCatalog' import ScenarioDetail from './components/Scenarios/ScenarioDetail' import ScenarioRunPage from './components/Scenarios/ScenarioRunPage' +import ScenarioPresetEditor from './components/ScenarioPresets/ScenarioPresetEditor' +import ScenarioPresetLibrary from './components/ScenarioPresets/ScenarioPresetLibrary' +import { presetEditorRoutePath } from './components/ScenarioPresets/presetRoutes' import FeedbackDialog from './components/Feedback/FeedbackDialog' import type { HistoryFilters } from './components/History/historyFilters' import { ConnectionBanner } from './components/ConnectionBanner' @@ -102,6 +105,11 @@ function LegacyScenarioRunRedirect() { return } +function LegacyPresetEditorRedirect() { + const { presetName } = useParams<{ presetName: string }>() + return +} + function LegacyScenarioHistoryRedirect() { const location = useLocation() return @@ -730,9 +738,25 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { } /> } /> + + } + /> + } /> + } /> } /> } /> + } /> + } /> + } /> }> Target registry content
} /> Converter registry content} /> + Preset library content} /> + Preset editor content} /> @@ -21,11 +23,12 @@ function renderLayout(initialPath = '/registry/targets') { } describe('RegistryLayout', () => { - it('shows target and converter registry tabs', () => { + it('shows target, converter and preset registry tabs', () => { renderLayout() expect(screen.getByRole('tab', { name: 'Targets' })).toHaveAttribute('aria-selected', 'true') expect(screen.getByRole('tab', { name: 'Converters' })).toHaveAttribute('aria-selected', 'false') + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'false') expect(screen.getByText('Target registry content')).toBeInTheDocument() }) @@ -38,4 +41,21 @@ describe('RegistryLayout', () => { expect(screen.getByRole('tab', { name: 'Converters' })).toHaveAttribute('aria-selected', 'true') expect(screen.getByText('Converter registry content')).toBeInTheDocument() }) + + it('opens the preset library from its tab', async () => { + const user = userEvent.setup() + renderLayout() + + await user.click(screen.getByRole('tab', { name: 'Scenario presets' })) + + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByText('Preset library content')).toBeInTheDocument() + }) + + it('keeps the preset tab selected on a nested preset route', () => { + renderLayout('/registry/scenario-presets/new') + + expect(screen.getByRole('tab', { name: 'Scenario presets' })).toHaveAttribute('aria-selected', 'true') + expect(screen.getByText('Preset editor content')).toBeInTheDocument() + }) }) diff --git a/frontend/src/components/Registry/RegistryLayout.tsx b/frontend/src/components/Registry/RegistryLayout.tsx index c1b943f3a1..a38b6e0914 100644 --- a/frontend/src/components/Registry/RegistryLayout.tsx +++ b/frontend/src/components/Registry/RegistryLayout.tsx @@ -3,11 +3,25 @@ import { Outlet, useLocation, useNavigate } from 'react-router' import { useRegistryLayoutStyles } from './Registry.styles' +const REGISTRY_TABS = ['targets', 'converters', 'scenario-presets'] as const + +/** + * Keeps the tab selected while a section owns nested routes, which the preset + * editor does — `/registry/scenario-presets/new` is still the presets tab. + */ +function selectedTabFromPath(pathname: string): string { + return ( + REGISTRY_TABS.find( + (tab) => pathname === `/registry/${tab}` || pathname.startsWith(`/registry/${tab}/`), + ) ?? 'targets' + ) +} + export default function RegistryLayout() { const styles = useRegistryLayoutStyles() const location = useLocation() const navigate = useNavigate() - const selectedTab = location.pathname.endsWith('/converters') ? 'converters' : 'targets' + const selectedTab = selectedTabFromPath(location.pathname) return (
@@ -20,6 +34,7 @@ export default function RegistryLayout() { > Targets Converters + Scenario presets
diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts new file mode 100644 index 0000000000..7f11684d49 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.styles.ts @@ -0,0 +1,20 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +export const useLaunchPresetDialogStyles = makeStyles({ + body: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + summary: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + }, + scenarioLine: { + color: tokens.colorNeutralForeground3, + }, + numberInput: { + maxWidth: '10rem', + }, +}) diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx new file mode 100644 index 0000000000..4b1d7f0cbf --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.test.tsx @@ -0,0 +1,308 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter } from 'react-router' + +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { makeTarget } from '@/test-utils/targetFixtures' +import type { ScenarioPreset, TargetInstance } from '@/types' + +import LaunchPresetDialog from './LaunchPresetDialog' + +jest.mock('@/services/api', () => ({ + scenarioPresetsApi: { + resolve: jest.fn(), + }, + scenariosApi: { + startRun: jest.fn(), + }, +})) + +const mockResolve = scenarioPresetsApi.resolve as jest.Mock +const mockStartRun = scenariosApi.startRun as jest.Mock + +const mockNavigate = jest.fn() + +jest.mock('react-router', () => ({ + ...jest.requireActual('react-router'), + useNavigate: () => mockNavigate, +})) + +const PRESET: ScenarioPreset = { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['crescendo'], +} + +const OBJECTIVE_TARGET = makeTarget({ target_registry_name: 'gpt4o' }) +const ADVERSARIAL_TARGET: TargetInstance = makeTarget({ + target_registry_name: 'adversary', + capabilities: { + supports_multi_turn: true, + supports_json_schema: false, + supports_json_output: false, + supports_system_prompt: true, + supported_input_modalities: ['text'], + supported_output_modalities: ['text'], + }, +}) + +interface RenderOptions { + preset?: ScenarioPreset + version?: string + defaultObjectiveTarget?: TargetInstance | null + defaultAdversarialTarget?: TargetInstance | null + labels?: Record +} + +const onDismiss = jest.fn() +const onPresetChanged = jest.fn() + +function renderDialog({ + preset = PRESET, + version = 'v1', + defaultObjectiveTarget = OBJECTIVE_TARGET, + defaultAdversarialTarget = null, + labels = {}, +}: RenderOptions = {}) { + return render( + + + + + , + ) +} + +beforeEach(() => { + jest.clearAllMocks() + mockResolve.mockResolvedValue({ scenario_name: 'foundry.red_team_agent', target_name: 'gpt4o' }) + mockStartRun.mockResolvedValue({ scenario_result_id: 'run-1' }) +}) + +describe('LaunchPresetDialog', () => { + it('reports scenario defaults for the fields the preset leaves unset', () => { + renderDialog() + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent( + '1 technique · scenario default datasets', + ) + }) + + it('summarizes the configuration the preset pins', () => { + renderDialog({ + preset: { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['crescendo', 'flip'], + dataset_names: ['harmbench', 'advbench'], + max_dataset_size: 25, + include_baseline: true, + }, + }) + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent( + '2 techniques · harmbench, advbench · max dataset size 25 · baseline included', + ) + }) + + it('distinguishes an excluded baseline from an included one', () => { + renderDialog({ preset: { ...PRESET, include_baseline: false } }) + + expect(screen.getByTestId('launch-preset-summary')).toHaveTextContent('baseline excluded') + }) + + it('resolves the preset server-side and navigates to the started run', async () => { + const user = userEvent.setup() + renderDialog({ labels: { op: 'nightly' } }) + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith('nightly_probe', { + expected_version: 'v1', + target_name: 'gpt4o', + max_concurrency: 10, + max_retries: 0, + labels: { op: 'nightly' }, + })) + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + target_name: 'gpt4o', + }) + expect(mockNavigate).toHaveBeenCalledWith('/scanner-history/run-1', { + state: { scenarioName: 'foundry.red_team_agent' }, + }) + }) + + it('omits the adversarial target and labels when neither is set', async () => { + const user = userEvent.setup() + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith('nightly_probe', { + expected_version: 'v1', + target_name: 'gpt4o', + max_concurrency: 10, + max_retries: 0, + })) + }) + + it('hands a preset edited mid-launch back to the library instead of running it', async () => { + const user = userEvent.setup() + mockResolve.mockRejectedValue({ + isAxiosError: true, + response: { status: 409, data: { detail: "Scenario preset 'nightly_probe' changed since it was read" } }, + }) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(onPresetChanged).toHaveBeenCalled()) + expect(mockStartRun).not.toHaveBeenCalled() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('sends the default adversarial target when one is configured', async () => { + const user = userEvent.setup() + renderDialog({ defaultAdversarialTarget: ADVERSARIAL_TARGET }) + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ adversarial_target_name: 'adversary' }), + )) + }) + + it('blocks launching until a target is chosen', () => { + renderDialog({ defaultObjectiveTarget: null }) + + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + }) + + it('ignores a default target that this deployment does not have', () => { + renderDialog({ defaultObjectiveTarget: makeTarget({ target_registry_name: 'retired' }) }) + + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + }) + + it('reports a resolve failure and does not start a run', async () => { + const user = userEvent.setup() + mockResolve.mockRejectedValue(new Error('Unknown scenario "foundry.retired".')) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + expect(await screen.findByText('Unknown scenario "foundry.retired".')).toBeInTheDocument() + expect(mockStartRun).not.toHaveBeenCalled() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('re-enables launch after a failure so the operator can retry', async () => { + const user = userEvent.setup() + mockStartRun.mockRejectedValueOnce(new Error('Target is unavailable.')) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + + expect(await screen.findByText('Target is unavailable.')).toBeInTheDocument() + expect(screen.getByTestId('confirm-launch-preset')).toBeEnabled() + + await user.click(screen.getByTestId('confirm-launch-preset')) + await waitFor(() => expect(mockNavigate).toHaveBeenCalled()) + }) + + it('dismisses without launching', async () => { + const user = userEvent.setup() + renderDialog() + + await user.click(screen.getByRole('button', { name: 'Cancel' })) + + expect(onDismiss).toHaveBeenCalled() + expect(mockResolve).not.toHaveBeenCalled() + }) + + it('dismisses when the dialog itself is closed', async () => { + const user = userEvent.setup() + renderDialog() + + await user.keyboard('{Escape}') + + expect(onDismiss).toHaveBeenCalled() + }) + + it('sends the target the operator picked instead of the default', async () => { + const user = userEvent.setup() + renderDialog() + + await user.selectOptions(screen.getByRole('combobox', { name: 'Target' }), 'adversary') + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ target_name: 'adversary' }), + )) + }) + + it('sends the adversarial target the operator picked', async () => { + const user = userEvent.setup() + renderDialog() + + await user.selectOptions(screen.getByRole('combobox', { name: 'Adversarial Target' }), 'adversary') + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ adversarial_target_name: 'adversary' }), + )) + }) + + it('sends the concurrency and retry limits the operator set', async () => { + const user = userEvent.setup() + renderDialog() + + const concurrency = screen.getByTestId('preset-max-concurrency-input') + await user.clear(concurrency) + await user.type(concurrency, '4') + const retries = screen.getByTestId('preset-max-retries-input') + await user.clear(retries) + await user.type(retries, '2') + + await user.click(screen.getByTestId('confirm-launch-preset')) + + await waitFor(() => expect(mockResolve).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ max_concurrency: 4, max_retries: 2 }), + )) + }) + + it('does not resolve twice while a launch is in flight', async () => { + const user = userEvent.setup() + let release: () => void = () => {} + mockResolve.mockReturnValue(new Promise((resolve) => { + release = () => resolve({ scenario_name: 'foundry.red_team_agent', target_name: 'gpt4o' }) + })) + + renderDialog() + + await user.click(screen.getByTestId('confirm-launch-preset')) + expect(screen.getByTestId('confirm-launch-preset')).toBeDisabled() + + release() + await waitFor(() => expect(mockNavigate).toHaveBeenCalled()) + expect(mockResolve).toHaveBeenCalledTimes(1) + }) +}) diff --git a/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx new file mode 100644 index 0000000000..7a36ce9439 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/LaunchPresetDialog.tsx @@ -0,0 +1,235 @@ +import { type FormEvent, useState } from 'react' + +import { + Button, + Dialog, + DialogActions, + DialogBody, + DialogContent, + DialogSurface, + DialogTitle, + Field, + MessageBar, + MessageBarBody, + Text, +} from '@fluentui/react-components' +import { useNavigate } from 'react-router' + +import TargetSelect from '@/components/Config/TargetSelect' +import SingleStepSpinButton from '@/components/Parameters/SingleStepSpinButton' +import { + DEFAULT_MAX_CONCURRENCY, + DEFAULT_MAX_RETRIES, + MAX_MAX_CONCURRENCY, + MAX_MAX_RETRIES, + MIN_MAX_CONCURRENCY, + MIN_MAX_RETRIES, + resolveSpinButtonValue, +} from '@/components/Scenarios/scenarioRunLimits' +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { ScenarioPreset, TargetInstance } from '@/types' +import { scenarioRunRoutePath } from '@/utils/routeParams' + +import { useLaunchPresetDialogStyles } from './LaunchPresetDialog.styles' + +interface LaunchPresetDialogProps { + preset: ScenarioPreset + version: string + targets: TargetInstance[] + defaultObjectiveTarget: TargetInstance | null + defaultAdversarialTarget: TargetInstance | null + labels: Record + onDismiss: () => void + onPresetChanged: () => void +} + +function initialTargetName( + candidate: TargetInstance | null, + available: TargetInstance[], +): string { + if (!candidate) { + return '' + } + const match = available.some( + (target) => target.target_registry_name === candidate.target_registry_name, + ) + return match ? candidate.target_registry_name : '' +} + +/** + * Restates what the preset pins. The dialog covers the library card, so without + * this the operator commits to a run knowing only the preset and scenario name. + * Omitted fields read as scenario defaults because that is what the server does + * with them. + */ +function presetSummary(preset: ScenarioPreset): string { + const techniques = preset.techniques ?? [] + const datasets = preset.dataset_names ?? [] + const parts = [ + techniques.length === 0 + ? 'Scenario default techniques' + : `${techniques.length} technique${techniques.length === 1 ? '' : 's'}`, + datasets.length === 0 ? 'scenario default datasets' : datasets.join(', '), + ] + if (preset.max_dataset_size != null) { + parts.push(`max dataset size ${preset.max_dataset_size}`) + } + if (preset.include_baseline != null) { + parts.push(preset.include_baseline ? 'baseline included' : 'baseline excluded') + } + return parts.join(' · ') +} + +/** + * Collects the launch-owned fields a preset deliberately omits, then asks the + * server to merge them with the stored preset. Resolution stays server-side so + * the browser never reimplements the preset-to-run mapping. + * + * The summary above describes the preset as it was read, so the launch sends that + * version back. An edit landing in between is reported rather than launched, which + * is the one case where a run would otherwise differ from what was confirmed. + */ +export default function LaunchPresetDialog({ + preset, + version, + targets, + defaultObjectiveTarget, + defaultAdversarialTarget, + labels, + onDismiss, + onPresetChanged, +}: LaunchPresetDialogProps) { + const styles = useLaunchPresetDialogStyles() + const navigate = useNavigate() + const adversarialTargets = targets.filter( + (target) => target.capabilities?.supports_multi_turn === true, + ) + const [targetName, setTargetName] = useState( + () => initialTargetName(defaultObjectiveTarget, targets), + ) + const [adversarialTargetName, setAdversarialTargetName] = useState( + () => initialTargetName(defaultAdversarialTarget, adversarialTargets), + ) + const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) + const [maxRetries, setMaxRetries] = useState(DEFAULT_MAX_RETRIES) + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + + const handleSubmit = async (event: FormEvent): Promise => { + event.preventDefault() + if (targetName === '' || submitting) { + return + } + setSubmitting(true) + setError(null) + try { + const request = await scenarioPresetsApi.resolve(preset.name, { + expected_version: version, + target_name: targetName, + ...(adversarialTargetName === '' ? {} : { adversarial_target_name: adversarialTargetName }), + max_concurrency: maxConcurrency, + max_retries: maxRetries, + ...(Object.keys(labels).length > 0 ? { labels } : {}), + }) + const summary = await scenariosApi.startRun(request) + navigate(scenarioRunRoutePath(summary.scenario_result_id), { + state: { scenarioName: preset.scenario_name }, + }) + } catch (err) { + const apiError = toApiError(err) + if (apiError.status === 409) { + // The summary on screen no longer describes the preset, so re-reading is the + // only way forward; keeping the dialog open would invite a confirm-and-retry loop. + onPresetChanged() + return + } + setError(apiError.detail) + setSubmitting(false) + } + } + + return ( + { if (!data.open) onDismiss() }}> + +
+ + Launch {preset.name} + +
+ + Runs {preset.scenario_name} with this preset's saved configuration. + + + {presetSummary(preset)} + +
+ {error && ( + + {error} + + )} + setTargetName(target?.target_registry_name ?? '')} + label="Target" + hint="The registered target this run attacks." + placeholder="Select a target" + disabled={submitting} + /> + setAdversarialTargetName(target?.target_registry_name ?? '')} + label="Adversarial Target" + hint="Only used by scenarios that generate attacks with a second target." + placeholder="Use server default" + disabled={submitting} + /> + + setMaxConcurrency(resolveSpinButtonValue(data, maxConcurrency))} + data-testid="preset-max-concurrency-input" + /> + + + setMaxRetries(resolveSpinButtonValue(data, maxRetries))} + data-testid="preset-max-retries-input" + /> + + + + + + + +
+ + + ) +} diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts new file mode 100644 index 0000000000..d48adda1f6 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.styles.ts @@ -0,0 +1,70 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' +import { WORKSPACE_CANVAS_BACKGROUND } from '@/styles/workspaceBackground' + +export const useScenarioPresetEditorStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + height: '100%', + width: '100%', + minWidth: 0, + padding: tokens.spacingVerticalXXL, + overflowX: 'hidden', + overflowY: 'auto', + backgroundColor: WORKSPACE_CANVAS_BACKGROUND, + [NARROW_VIEWPORT_QUERY]: { + padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalM}`, + }, + }, + headerText: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + marginBottom: tokens.spacingVerticalL, + }, + subtitle: { + color: tokens.colorNeutralForeground3, + }, + form: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalL, + maxWidth: '60rem', + }, + section: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + padding: tokens.spacingVerticalL, + borderRadius: tokens.borderRadiusMedium, + border: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + backgroundColor: tokens.colorNeutralBackground1, + }, + control: { + maxWidth: '32rem', + }, + dynamicParameters: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + }, + actions: { + display: 'flex', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + alignItems: 'center', + }, + touchTarget: { + ...mobileTouchTarget, + }, + centeredState: { + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: tokens.spacingVerticalM, + padding: tokens.spacingVerticalXXL, + textAlign: 'center', + }, +}) diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx new file mode 100644 index 0000000000..2d919608c8 --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx @@ -0,0 +1,524 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter, Route, Routes } from 'react-router' + +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { makeScenario } from '@/test-utils/scenarioFixtures' +import type { ScenarioPreset } from '@/types' + +import ScenarioPresetEditor from './ScenarioPresetEditor' + +jest.mock('@/services/api', () => ({ + scenarioPresetsApi: { + get: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + scenariosApi: { + listCatalog: jest.fn(), + getScenario: jest.fn(), + }, +})) + +const mockGet = scenarioPresetsApi.get as jest.Mock +const mockCreate = scenarioPresetsApi.create as jest.Mock +const mockUpdate = scenarioPresetsApi.update as jest.Mock +const mockListCatalog = scenariosApi.listCatalog as jest.Mock +const mockGetScenario = scenariosApi.getScenario as jest.Mock + +const mockNavigate = jest.fn() + +jest.mock('react-router', () => ({ + ...jest.requireActual('react-router'), + useNavigate: () => mockNavigate, +})) + +const SCENARIO = makeScenario() + +const STORED_PRESET: ScenarioPreset = { + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + description: 'Nightly smoke test.', + techniques: ['crescendo'], + include_baseline: false, +} + +function apiError(status: number, detail: string): unknown { + return { + isAxiosError: true, + response: { status, data: { detail } }, + } +} + +function renderCreate() { + return render( + + + + } /> + + + , + ) +} + +function renderEdit(name = 'nightly_probe') { + return render( + + + + } + /> + + + , + ) +} + +beforeEach(() => { + jest.clearAllMocks() + mockListCatalog.mockResolvedValue({ + items: [SCENARIO], + pagination: { has_more: false, next_cursor: null }, + }) + mockGetScenario.mockResolvedValue(SCENARIO) + mockGet.mockResolvedValue({ preset: STORED_PRESET, version: 'v1', issues: [] }) + mockCreate.mockResolvedValue({ preset: STORED_PRESET, version: 'v1', issues: [] }) + mockUpdate.mockResolvedValue({ preset: STORED_PRESET, version: 'v2', issues: [] }) +}) + +describe('ScenarioPresetEditor create mode', () => { + it('cannot save before a scenario is chosen', async () => { + renderCreate() + + expect(await screen.findByTestId('scenario-preset-editor')).toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) + + it('creates a preset that pins nothing the operator left at the scenario default', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockCreate).toHaveBeenCalledWith({ + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + })) + expect(mockUpdate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('pins a technique selection the operator moved off the scenario default', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await user.click(await screen.findByTestId('technique-crescendo')) + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockCreate).toHaveBeenCalledWith({ + name: 'nightly_probe', + scenario_name: 'foundry.red_team_agent', + techniques: ['default_technique', 'crescendo'], + })) + }) + + it('rejects a name the server pattern would reject, without calling the API', async () => { + const user = userEvent.setup() + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'Nightly-Probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText(/Use lowercase letters/)).toBeInTheDocument() + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('reports a duplicate name returned by the server', async () => { + const user = userEvent.setup() + mockCreate.mockRejectedValue(apiError(409, 'A preset named "nightly_probe" already exists.')) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect( + await screen.findByText('A preset named "nightly_probe" already exists.'), + ).toBeInTheDocument() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('reports a missing admin permission', async () => { + const user = userEvent.setup() + mockCreate.mockRejectedValue(apiError(403, 'Admin access required.')) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText('Admin access required.')).toBeInTheDocument() + }) + + it('reports an unselectable technique set rather than saving an empty one', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(makeScenario({ + default_techniques: [], + default_technique: 'all', + aggregate_techniques: ['all'], + })) + renderCreate() + + await user.type(await screen.findByTestId('preset-name-input'), 'nightly_probe') + await user.click(screen.getByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + await waitFor(() => expect(screen.getByTestId('save-preset-btn')).toBeEnabled()) + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText('Select at least one technique.')).toBeInTheDocument() + expect(mockCreate).not.toHaveBeenCalled() + }) + + it('reports a failure to load the scenario the operator picked', async () => { + const user = userEvent.setup() + mockGetScenario.mockRejectedValue(apiError(503, 'Scenario registry is unavailable.')) + renderCreate() + + await user.click(await screen.findByTestId('preset-scenario-select')) + await user.click(await screen.findByRole('option', { name: 'foundry.red_team_agent' })) + + expect(await screen.findByText('Scenario registry is unavailable.')).toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) +}) + +describe('ScenarioPresetEditor edit mode', () => { + it('loads the stored preset and pins its name and scenario', async () => { + renderEdit() + + expect(await screen.findByTestId('scenario-preset-editor')).toBeInTheDocument() + expect(screen.getByTestId('preset-name-input')).toBeDisabled() + expect(screen.getByTestId('preset-name-input')).toHaveValue('nightly_probe') + expect(screen.getByTestId('preset-description-input')).toHaveValue('Nightly smoke test.') + }) + + it('updates with the version it read so a concurrent edit is not overwritten', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ name: 'nightly_probe', techniques: ['crescendo'] }), + 'v1', + )) + expect(mockCreate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('reports a version conflict instead of navigating away', async () => { + const user = userEvent.setup() + mockUpdate.mockRejectedValue(apiError(409, 'The preset changed since it was loaded.')) + renderEdit() + + await user.click(await screen.findByTestId('save-preset-btn')) + + expect(await screen.findByText('The preset changed since it was loaded.')).toBeInTheDocument() + expect(mockNavigate).not.toHaveBeenCalled() + }) + + it('keeps pinned techniques this deployment does not offer instead of rewriting them', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['crescendo', 'retired_attack'] }, + version: 'v1', + issues: [], + }) + + renderEdit() + + expect(await screen.findByTestId('dropped-techniques-warning')).toHaveTextContent( + 'retired_attack', + ) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo', 'retired_attack'] }), + 'v1', + )) + }) + + it('keeps a converter-qualified technique through an unrelated edit', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['crescendo:converter.translation_spanish'] }, + version: 'v1', + issues: [], + }) + + renderEdit() + + await user.type(await screen.findByTestId('preset-description-input'), ' Updated.') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo:converter.translation_spanish'] }), + 'v1', + )) + }) + + it('shows a pinned aggregate technique as a tag the operator can remove', async () => { + const user = userEvent.setup() + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, techniques: ['all'] }, + version: 'v1', + issues: [], + }) + + renderEdit() + + expect(await screen.findByTestId('techniques-without-checkbox')).toHaveTextContent('all') + expect(screen.getByTestId('technique-crescendo')).not.toBeChecked() + expect(screen.queryByTestId('dropped-techniques-warning')).not.toBeInTheDocument() + + await user.click(screen.getByTestId('technique-crescendo')) + await user.click(screen.getByRole('button', { name: 'Remove all' })) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ techniques: ['crescendo'] }), + 'v1', + )) + }) + + it('reports a missing preset as not found', async () => { + mockGet.mockRejectedValue(apiError(404, 'No such preset.')) + + renderEdit() + + expect(await screen.findByTestId('editor-error-state')).toHaveTextContent( + 'No preset named "nightly_probe"', + ) + }) + + it('reports an unavailable scenario as unavailable, not as a missing preset', async () => { + mockGetScenario.mockRejectedValue(apiError(404, 'Unknown scenario "foundry.red_team_agent".')) + + renderEdit() + + expect(await screen.findByTestId('scenario-unavailable-warning')).toBeInTheDocument() + expect(screen.queryByTestId('editor-error-state')).not.toBeInTheDocument() + expect(screen.getByTestId('save-preset-btn')).toBeDisabled() + }) + + it('decodes a preset name that was escaped into the route', async () => { + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, name: 'a_b' }, + version: 'v1', + issues: [], + }) + + renderEdit('a_b') + + await waitFor(() => expect(mockGet).toHaveBeenCalledWith('a_b')) + }) + + it('saves the edited description', async () => { + const user = userEvent.setup() + renderEdit() + + const description = await screen.findByTestId('preset-description-input') + await user.clear(description) + await user.type(description, 'Weekly smoke test.') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ description: 'Weekly smoke test.' }), + 'v1', + )) + }) + + it('saves the techniques and baseline choice the operator changed', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByTestId('technique-default_technique')) + await user.click(screen.getByTestId('baseline-checkbox')) + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ + techniques: ['crescendo', 'default_technique'], + include_baseline: true, + }), + 'v1', + )) + }) + + it('saves the dataset overrides the operator entered', async () => { + const user = userEvent.setup() + renderEdit() + + await user.type(await screen.findByTestId('dataset-override-input'), 'harmbench, xstest') + await user.type(screen.getByTestId('max-dataset-size-input'), '25') + await user.type(screen.getByTestId('harm-categories-filter-input'), 'violence') + await user.type(screen.getByTestId('data-types-filter-input'), 'text') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ + dataset_names: ['harmbench', 'xstest'], + max_dataset_size: 25, + dataset_filters: { harm_categories: ['violence'], data_types: ['text'] }, + }), + 'v1', + )) + }) + + it('rejects a non-positive dataset size before calling the server', async () => { + const user = userEvent.setup() + renderEdit() + + await user.type(await screen.findByTestId('max-dataset-size-input'), '0') + await user.click(screen.getByTestId('save-preset-btn')) + + expect(await screen.findByText('Max dataset size must be a positive integer.')).toBeInTheDocument() + expect(mockUpdate).not.toHaveBeenCalled() + }) + + it('cannot pin a dataset cap on a scenario that sizes itself by prompt generation', async () => { + const generativeScenario = makeScenario({ + default_run_size: { + dataset_size: { kind: 'indeterminate', detail: 'Generated at run time.' }, + dataset_limit: { state: 'not_applicable' }, + estimated_attack_count: null, + components: [], + datasets: [], + note: null, + }, + }) + mockGetScenario.mockResolvedValue(generativeScenario) + renderEdit() + + expect(await screen.findByTestId('max-dataset-size-input')).toBeDisabled() + expect( + screen.getByText('This scenario uses prompt-generation limits instead of a dataset size limit.'), + ).toBeInTheDocument() + }) + + it('leaves the editor without saving when cancelled', async () => { + const user = userEvent.setup() + renderEdit() + + await user.click(await screen.findByRole('button', { name: 'Cancel' })) + + expect(mockUpdate).not.toHaveBeenCalled() + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('returns to the library from the not-found state', async () => { + const user = userEvent.setup() + mockGet.mockRejectedValue(apiError(404, 'No such preset.')) + + renderEdit() + + await user.click(await screen.findByRole('button', { name: 'Back to presets' })) + + expect(mockNavigate).toHaveBeenCalledWith('/registry/scenario-presets') + }) + + it('reports a catalog failure that is not a missing preset as an error', async () => { + mockListCatalog.mockRejectedValue(apiError(503, 'Preset storage is not configured.')) + + renderEdit() + + expect(await screen.findByTestId('editor-error-state')).toHaveTextContent( + 'Preset storage is not configured.', + ) + }) +}) + +describe('ScenarioPresetEditor dynamic parameters', () => { + const SCENARIO_WITH_PARAM = makeScenario({ + supported_parameters: [ + { + name: 'max_turns', + type_name: 'int', + required: false, + default: '5', + description: 'Turn budget.', + }, + ], + }) + + it('stores a scenario-specific parameter the operator changed', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(SCENARIO_WITH_PARAM) + renderEdit() + + const field = await screen.findByTestId('preset-param-max_turns') + await user.clear(field) + await user.type(field, '9') + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ scenario_params: { max_turns: 9 } }), + 'v1', + )) + }) + + it('preserves and surfaces stored parameters this editor renders no field for', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(SCENARIO_WITH_PARAM) + mockGet.mockResolvedValue({ + preset: { ...STORED_PRESET, scenario_params: { max_turns: 5, max_concurrency: 4 } }, + version: 'v1', + issues: [], + }) + + renderEdit() + + expect(await screen.findByTestId('carried-params-notice')).toHaveTextContent('max_concurrency') + + await user.click(screen.getByTestId('save-preset-btn')) + + await waitFor(() => expect(mockUpdate).toHaveBeenCalledWith( + 'nightly_probe', + expect.objectContaining({ scenario_params: { max_turns: 5, max_concurrency: 4 } }), + 'v1', + )) + }) +}) diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx new file mode 100644 index 0000000000..38d04b8c1e --- /dev/null +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.tsx @@ -0,0 +1,436 @@ +import { type FormEvent, useCallback, useEffect, useMemo, useState } from 'react' + +import { + Button, + Field, + Input, + MessageBar, + MessageBarBody, + Option, + Spinner, + Combobox, + Text, + Textarea, +} from '@fluentui/react-components' +import { useNavigate, useParams } from 'react-router' + +import ParameterField from '@/components/Parameters/ParameterField' +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' +import ScenarioDatasetFields from '@/components/Scenarios/ScenarioDatasetFields' +import ScenarioTechniqueSelector from '@/components/Scenarios/ScenarioTechniqueSelector' +import { + buildScenarioConfig, + defaultMaxDatasetSize, + dynamicScenarioParameters, + uniqueTechniqueOptions, + type ScenarioConfigFormState, +} from '@/components/Scenarios/scenarioConfigForm' +import { scenarioPresetsApi, scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { RegisteredScenario, ScenarioPreset } from '@/types' +import { fetchAllPages } from '@/utils/fetchAllPages' +import { routerPathParamValue } from '@/utils/routeParams' + +import { useScenarioPresetEditorStyles } from './ScenarioPresetEditor.styles' +import { PRESETS_ROUTE } from './presetRoutes' +import { + configToPreset, + initialPresetConfigState, + presetToConfigState, + uneditableScenarioParams, + unknownPresetTechniques, + validatePresetName, +} from './scenarioPresetForm' + +/** Items requested per catalog page while paging the scenario picker's options. */ +const CATALOG_PAGE_SIZE = 200 + +type LoadStatus = 'loading' | 'ready' | 'not-found' | 'error' + +interface LoadedPreset { + preset: ScenarioPreset + version: string +} + +export default function ScenarioPresetEditor({ mode }: { mode: 'create' | 'edit' }) { + const { presetName } = useParams<{ presetName: string }>() + // Remount on navigation between two presets so every field resets to the + // newly loaded document rather than retaining the previous one's edits. + return +} + +interface ScenarioPresetEditorContentProps { + mode: 'create' | 'edit' + presetName: string | undefined +} + +function ScenarioPresetEditorContent({ mode, presetName }: ScenarioPresetEditorContentProps) { + const styles = useScenarioPresetEditorStyles() + const navigate = useNavigate() + const decodedName = presetName === undefined ? '' : routerPathParamValue(presetName) + + const [status, setStatus] = useState('loading') + const [loadError, setLoadError] = useState(null) + const [catalog, setCatalog] = useState([]) + const [loaded, setLoaded] = useState(null) + + const [name, setName] = useState(mode === 'edit' ? decodedName : '') + const [description, setDescription] = useState('') + const [scenario, setScenario] = useState(null) + const [scenarioLoading, setScenarioLoading] = useState(false) + const [config, setConfig] = useState(null) + const [droppedTechniques, setDroppedTechniques] = useState([]) + const [scenarioUnavailable, setScenarioUnavailable] = useState(null) + + const [saving, setSaving] = useState(false) + const [validationError, setValidationError] = useState(null) + const [saveError, setSaveError] = useState(null) + + useEffect(() => { + let cancelled = false + + const load = async (): Promise => { + try { + const scenarios = await fetchAllPages( + (cursor) => scenariosApi.listCatalog(CATALOG_PAGE_SIZE, cursor, false), + undefined, + (entry) => entry.scenario_name, + ) + if (cancelled) return + setCatalog(scenarios) + + if (mode === 'create') { + setStatus('ready') + return + } + + const response = await scenarioPresetsApi.get(decodedName) + if (cancelled) return + setLoaded({ preset: response.preset, version: response.version }) + setName(response.preset.name) + setDescription(response.preset.description ?? '') + setStatus('ready') + + // The list endpoint omits run-size estimates, so the preset's scenario is + // re-fetched in full to get the configured dataset caps the form shows. + // A preset may legitimately name a scenario this deployment lacks; that + // is an unavailable scenario, not a missing preset, so it must not be + // reported as a 404 on the preset itself. + try { + const full = await scenariosApi.getScenario(response.preset.scenario_name) + if (cancelled) return + setScenario(full) + setConfig(presetToConfigState(full, response.preset)) + setDroppedTechniques(unknownPresetTechniques(full, response.preset)) + } catch (scenarioErr) { + if (cancelled) return + setScenarioUnavailable(toApiError(scenarioErr).detail) + } + } catch (err) { + if (cancelled) return + const apiError = toApiError(err) + setLoadError(apiError.detail) + setStatus(apiError.status === 404 ? 'not-found' : 'error') + } + } + + void load() + return () => { + cancelled = true + } + }, [decodedName, mode]) + + const handleScenarioChange = useCallback(async (nextScenarioName: string): Promise => { + setScenarioLoading(true) + setSaveError(null) + setValidationError(null) + setDroppedTechniques([]) + try { + const full = await scenariosApi.getScenario(nextScenarioName) + setScenario(full) + setConfig(initialPresetConfigState(full)) + } catch (err) { + setSaveError(toApiError(err).detail) + } finally { + setScenarioLoading(false) + } + }, []) + + const dynamicParameters = useMemo( + () => (scenario ? dynamicScenarioParameters(scenario) : []), + [scenario], + ) + const techniqueOptions = useMemo( + () => (scenario ? uniqueTechniqueOptions(scenario).techniques : []), + [scenario], + ) + const carriedScenarioParams = useMemo( + () => (scenario ? Object.keys(uneditableScenarioParams(scenario, loaded?.preset ?? null)) : []), + [loaded, scenario], + ) + + const updateConfig = (patch: Partial): void => { + setConfig((current) => (current ? { ...current, ...patch } : current)) + setValidationError(null) + } + + const updateScenarioParam = useCallback((parameterName: string, value: ParameterFormValue) => { + setConfig((current) => (current + ? { ...current, scenarioParamValues: { ...current.scenarioParamValues, [parameterName]: value } } + : current)) + setValidationError(null) + }, []) + + const handleSubmit = async (event: FormEvent): Promise => { + event.preventDefault() + if (saving) { + return + } + setSaveError(null) + + const trimmedName = name.trim() + const nameError = validatePresetName(trimmedName) + if (nameError) { + setValidationError(nameError) + return + } + if (!scenario || !config) { + setValidationError('Select a scenario.') + return + } + const built = buildScenarioConfig({ + techniques: config.techniques, + dynamicParameters, + scenarioParamValues: config.scenarioParamValues, + datasetOverride: config.datasetOverride, + maxDatasetSize: config.maxDatasetSize, + harmCategoriesFilter: config.harmCategoriesFilter, + dataTypesFilter: config.dataTypesFilter, + includeBaseline: config.includeBaseline, + }) + if (!built.ok) { + setValidationError(built.error) + return + } + setValidationError(null) + + const preset = configToPreset( + { name: trimmedName, scenarioName: scenario.scenario_name, description }, + built.config, + { scenario, previous: loaded?.preset ?? null }, + ) + + setSaving(true) + try { + if (loaded) { + await scenarioPresetsApi.update(loaded.preset.name, preset, loaded.version) + } else { + await scenarioPresetsApi.create(preset) + } + navigate(PRESETS_ROUTE) + } catch (err) { + setSaveError(toApiError(err).detail) + setSaving(false) + } + } + + if (status === 'loading') { + return ( +
+
+ +
+
+ ) + } + + if (status !== 'ready') { + return ( +
+
+ + + {status === 'not-found' ? `No preset named "${decodedName}".` : loadError} + + + +
+
+ ) + } + + const editing = loaded !== null + + return ( +
+
+ + {editing ? `Edit ${decodedName}` : 'New preset'} + + + A preset stores what to test. The target, concurrency and retries are chosen at launch. + +
+ +
+ {saveError && ( + + {saveError} + + )} + {validationError && ( + + {validationError} + + )} + {droppedTechniques.length > 0 && ( + + + This preset pins techniques this deployment does not offer + ({droppedTechniques.join(', ')}). They are preserved on save — remove them + below if this preset should stop running them. + + + )} + {carriedScenarioParams.length > 0 && ( + + + This preset carries parameters this editor has no field for + ({carriedScenarioParams.join(', ')}). They are preserved as-is on save. + + + )} + {scenarioUnavailable && ( + + + This preset targets {loaded?.preset.scenario_name}, which is not available here + ({scenarioUnavailable}). Its configuration cannot be edited in this deployment. + + + )} + +
+ + { + setName(data.value) + setValidationError(null) + }} + data-testid="preset-name-input" + /> + + +