diff --git a/server/src/addie/jobs/compliance-heartbeat.ts b/server/src/addie/jobs/compliance-heartbeat.ts index d7e73dfc2d4..c51155d5639 100644 --- a/server/src/addie/jobs/compliance-heartbeat.ts +++ b/server/src/addie/jobs/compliance-heartbeat.ts @@ -1,3 +1,4 @@ +import { complianceRunProvenance } from '../../compliance/run-provenance.js'; import { isAuthoritativeComplianceRun } from '../../compliance/run-publication.js'; /** * Compliance Heartbeat Job @@ -19,6 +20,7 @@ import { selectedComplianceTargetMatchesObservedProfile, type ComplyOptions, type ComplianceTargetSelection, + type ComplianceResult, } from '../services/compliance-testing.js'; import { ComplianceDatabase, type LifecycleStage } from '../../db/compliance-db.js'; import { query, withDatabaseDeadline } from '../../db/client.js'; @@ -459,6 +461,7 @@ export async function runComplianceHeartbeatJob(options: HeartbeatOptions = {}): triggered_by: 'heartbeat', dry_run: false, completeness: 'not_completed', + provenance_json: complianceRunProvenance({ adcp_version: runTarget.version, agent_profile: {} as ComplianceResult['agent_profile'] }), is_authoritative: false, replace_storyboard_statuses: true, }); diff --git a/server/src/addie/services/compliance-testing.ts b/server/src/addie/services/compliance-testing.ts index 32a3442a936..f1898ad2b5f 100644 --- a/server/src/addie/services/compliance-testing.ts +++ b/server/src/addie/services/compliance-testing.ts @@ -1,3 +1,4 @@ +import { complianceRunProvenance, type ComplianceRunProvenance } from '../../compliance/run-provenance.js'; /** * Compliance testing — thin adapter over @adcp/sdk's compliance module. * @@ -11,6 +12,7 @@ import { setAgentTesterLogger, comply as sdkComply, loadComplianceIndex as sdkLoadComplianceIndex, + getComplianceStoryboardById, testCapabilityDiscovery, CapabilityResolutionError, type ComplyOptions, @@ -22,6 +24,7 @@ import { } from '@adcp/sdk/testing'; import { hostedComplianceTarget, + hostedComplianceOptions, hostedAuthProbeTaskForProfile, hostedStaticApiKeyForProfile, agentAdvertisesHostedComplianceTarget, @@ -33,7 +36,7 @@ import { withHostedTestOptions, type HostedComplianceTarget, } from '../../services/hosted-compliance-version.js'; -import { getStoryboard } from '../../services/storyboards.js'; +import { classifyComplianceStep } from '../../compliance/step-disposition.js'; import { createLogger } from '../../logger.js'; import { withSdkSafeTransport } from '../../utils/sdk-safe-fetch.js'; @@ -255,6 +258,8 @@ export async function comply( ), ); result.adcp_version ??= target.version; + (result as ComplianceResult & { hosted_provenance: ComplianceRunProvenance }).hosted_provenance = + complianceRunProvenance(result, options); (result as ComplianceResult & { requested_compliance_target?: string }).requested_compliance_target = target.requested; return result; } @@ -702,169 +707,139 @@ function mapOverallStatus(status: string): OverallRunStatus { } } -function isSyntheticRequiredToolsMissingSkip( - step: { skip_reason?: string; step?: unknown; step_id?: unknown }, - scenario: unknown, -): boolean { - if (step.skip_reason !== 'missing_tool' || typeof scenario !== 'string' || !scenario.endsWith('/missing_tool')) { - return false; - } - const stepId = firstString(step.step_id); - const title = firstString(step.step); - return stepId === 'missing_tool' || title?.startsWith('Skipped — agent does not advertise') === true; -} +const pinnedStoryboardContexts = new WeakMap>>(); -function isExplicitRequiresToolMissingSkip(step: { - skip_reason?: string; - details?: unknown; - error?: unknown; - warnings?: unknown; - skip?: { detail?: unknown }; -}): boolean { - if (step.skip_reason !== 'missing_tool') return false; - const warning = Array.isArray(step.warnings) - ? step.warnings.find((w): w is string => typeof w === 'string' && w.trim().length > 0) - : undefined; - const detail = firstString(step.skip?.detail, step.details, step.error, warning); - return detail?.startsWith('Required tool "') === true && detail.includes('" not advertised'); -} - -function isRunnerApplicabilitySkip(step: { - skip_reason?: string; - step_id?: unknown; - requirement?: unknown; -}, scenario?: unknown): boolean { - switch (step.skip_reason) { - case 'capability_unsupported': - case 'capability_prerequisite_unavailable': - return true; - case 'missing_test_kit_contract': - return scenario === 'idempotency/rate_limit_replay_invariant' && - firstString(step.step_id) === 'expect_rate_limit_not_replayed'; - case 'requirement_unmet': - return firstString(step.requirement) === 'webhook_receiver'; - default: - return false; +function pinnedStoryboard(result: ComplianceResult, storyboardId: string) { + if (!result.adcp_version) return undefined; + let context = pinnedStoryboardContexts.get(result); + if (!context) { + context = new Map(); + pinnedStoryboardContexts.set(result, context); } -} - -type SkipEvidenceStep = { - skip_reason?: string; - details?: unknown; - error?: unknown; - warnings?: unknown; - skip?: { detail?: unknown }; -}; - -const NO_SKIP_REASONS: ReadonlySet = new Set(); - -function prerequisiteCascadeSource(step: SkipEvidenceStep): { reason: string } | null { - if (step.skip_reason !== 'prerequisite_failed') return null; - const warning = Array.isArray(step.warnings) - ? step.warnings.find((value): value is string => typeof value === 'string' && value.trim().length > 0) - : undefined; - const detail = firstString(step.skip?.detail, step.error, step.details, warning); - const match = detail?.match(/prior stateful step "([^"]+)" skipped \(([^)]+)\)/); - return match ? { reason: match[2] } : null; -} - -function isApplicabilityCascade( - step: SkipEvidenceStep, - applicabilityReasons: ReadonlySet, - nonApplicabilityReasons: ReadonlySet = new Set(), -): boolean { - const source = prerequisiteCascadeSource(step); - if (!source) return false; - // A controller absence is always runner-owned applicability, including - // cross-phase cascades where the originating step is not in this slice. - return source.reason === 'missing_test_controller' || ( - applicabilityReasons.has(source.reason) && !nonApplicabilityReasons.has(source.reason) - ); + const key = `${result.adcp_version}:${storyboardId}`; + if (!context.has(key)) { + try { + context.set(key, getComplianceStoryboardById(storyboardId, + hostedComplianceOptions(hostedComplianceTarget(result.adcp_version)))); + } catch { + context.set(key, undefined); + } + } + return context.get(key); +} + +function classifyRunStep(result: ComplianceResult, scenario: string, step: Parameters[0] & { step?: string }) { + let optionalToolUnavailable = false; + if (step.skipped && ['missing_tool', 'prerequisite_failed', 'missing_test_kit_contract'].includes(step.skip_reason ?? '') && result.adcp_version) { + try { + const storyboardId = scenarioStoryboardIdForFallback(scenario); + if (!storyboardId) return classifyComplianceStep(step, scenario); + const phaseId = phaseIdFromScenario(scenario, storyboardId); + const storyboard = pinnedStoryboard(result, storyboardId); + const phase = storyboard?.phases.find(candidate => candidate.id === phaseId); + // rc.33 drops step IDs, so permit only an unambiguous title+task match in + // the exact version/phase. Never infer optionality from warning strings. + const matches = phase?.steps.filter(candidate => step.step_id + ? candidate.id === step.step_id + : candidate.title === step.step && candidate.task === step.task) ?? []; + if (step.skip_reason === 'missing_test_kit_contract' && matches.length === 1 && matches[0].requires_contract) { + return 'setup_gap'; + } + const requiredTool = matches.length === 1 ? matches[0].requires_tool : undefined; + optionalToolUnavailable = typeof requiredTool === 'string' && Array.isArray(result.agent_profile?.tools) && + !result.agent_profile.tools.includes(requiredTool); + } catch { + // Missing pinned metadata cannot turn a production tool failure into N/A. + } + } + if (optionalToolUnavailable) return 'not_applicable'; + if (step.skipped && step.skip_reason === 'prerequisite_failed') { + // fixture_unavailable is the SDK's whole-storyboard preflight abort. Once + // emitted, subsequent legacy cascades did not exercise seller behavior. + const storyboardId = scenarioStoryboardIdForFallback(scenario); + let fixtureAborted = false; + for (const track of result.tracks) { + for (const phase of track.scenarios) { + if (scenarioStoryboardIdForFallback(phase.scenario) !== storyboardId) continue; + for (const prior of phase.steps ?? []) { + if (prior === step) return fixtureAborted ? 'setup_gap' : classifyComplianceStep(step, scenario); + if (prior.skipped && prior.skip_reason === 'fixture_unavailable') fixtureAborted = true; + } + } + } + } + return classifyComplianceStep(step, scenario); } export function isNonExecutableCoverageGapScenario(scenario: { scenario?: unknown; - steps?: Array<{ - passed?: boolean; - skipped?: boolean; - skip_reason?: string; - step_id?: unknown; - requirement?: unknown; - details?: unknown; - error?: unknown; - warnings?: unknown; - skip?: { detail?: unknown }; - }>; + steps?: Array[0]>; }): boolean { - const steps = Array.isArray(scenario.steps) ? scenario.steps : []; - const scenarioId = typeof scenario.scenario === 'string' ? scenario.scenario : ''; - const hasGenuineFailure = steps.some((step) => !step.skipped && step.passed === false); - if (!hasGenuineFailure && steps.some( - (step) => step.skipped && step.skip_reason === 'fixture_unavailable', - )) return true; - return steps.length > 0 && steps.every((step) => { - if (!step?.skipped) return false; - const isApplicabilitySkip = step.skip_reason === 'peer_branch_taken' || - step.skip_reason === 'peer_substituted' || - step.skip_reason === 'missing_test_controller' || - step.skip_reason === 'fixture_unavailable' || - (step.skip_reason === 'requirement_unmet' && step.requirement === 'controller') || - isExplicitRequiresToolMissingSkip(step) || - isRunnerApplicabilitySkip(step, scenarioId); - if (isApplicabilitySkip) { - return true; - } - // This UI helper receives one phase without prior-phase provenance. - // Only controller absence is intrinsically safe to classify as N/A; - // other same-reason cascades stay visible as executable coverage gaps. - return isApplicabilityCascade(step, NO_SKIP_REASONS); + const steps = scenario.steps ?? []; + const dispositions = steps.map(step => classifyComplianceStep(step, String(scenario.scenario ?? ''))); + if (steps.some(step => step.skipped && step.skip_reason === 'fixture_unavailable') && + !dispositions.some(d => d === 'failed' || d === 'dependency_failed')) return true; + return steps.length > 0 && steps.every(step => { + const disposition = classifyComplianceStep(step, String(scenario.scenario ?? '')); + return disposition === 'not_applicable' || disposition === 'setup_gap'; }); } -function skipReasonIsCoverageGap( - reason: string | undefined, - step?: { - skip_reason?: string; - step?: unknown; - step_id?: unknown; - requirement?: unknown; - details?: unknown; - error?: unknown; - warnings?: unknown; - skip?: { detail?: unknown }; - }, - scenario?: unknown, -): boolean { - if (step && isSyntheticRequiredToolsMissingSkip(step, scenario)) return false; - if (step && isExplicitRequiresToolMissingSkip(step)) return false; - if (step && isRunnerApplicabilitySkip(step, scenario)) return false; - switch (reason) { - case 'not_applicable': - case 'peer_branch_taken': - case 'peer_substituted': - return false; - default: - return true; - } -} - -function trackHasCoverageGapSkip(track: TrackResult): boolean { - for (const scenario of track.scenarios) { - for (const step of scenario.steps ?? []) { - if (step.skipped && skipReasonIsCoverageGap(step.skip_reason, step, scenario.scenario)) { - return true; - } - } - } - return false; +function trackHasCoverageGapSkip(result: ComplianceResult, track: TrackResult): boolean { + return track.scenarios.some(scenario => scenario.steps?.some(step => + classifyRunStep(result, String(scenario.scenario), step) === 'setup_gap')); } function trackHasFixtureUnavailableSkip(track: TrackResult): boolean { - return track.scenarios.some((scenario) => - (scenario.steps ?? []).some((step) => - step.skipped && step.skip_reason === 'fixture_unavailable' - ) - ); + return track.scenarios.some(scenario => scenario.steps?.some(step => + step.skipped && step.skip_reason === 'fixture_unavailable')); +} + +/** Reconcile SDK aggregates with the same structural policy as storyboard rows. */ +function normalizeGradedTracks(result: ComplianceResult): ComplianceResult { + const tracks = result.tracks.map(track => { + // Preserve legacy phase-only results, including their failure verdicts. + if (!track.scenarios.some(scenario => scenario.steps?.length)) return track; + const phases = track.scenarios.map(scenario => { + if (!scenario.steps?.length) return { + scenario, + dispositions: [scenario.overall_passed ? 'passed' : 'failed'], + }; + const dispositions = scenario.steps.map(step => classifyRunStep(result, String(scenario.scenario), step)); + const hasFailure = dispositions.some(d => d === 'failed' || d === 'dependency_failed'); + const fixtureAborted = scenario.steps.some(step => step.skipped && step.skip_reason === 'fixture_unavailable'); + return { + scenario: { ...scenario, overall_passed: !hasFailure }, + dispositions: fixtureAborted && !hasFailure ? [] : dispositions, + }; + }); + const scenarios = phases.map(phase => phase.scenario); + const gradedSteps = phases.flatMap(phase => phase.dispositions); + const hasPassed = gradedSteps.includes('passed'); + const hasFailed = gradedSteps.some(d => d === 'failed' || d === 'dependency_failed') || + [...(track.observations ?? []), ...(result.observations ?? []).filter(observation => observation.track === track.track)] + .some(observation => observation.source?.code === 'storyboard-assertion-failed'); + const status: TrackResult['status'] = hasFailed ? hasPassed ? 'partial' : 'fail' + : !hasPassed ? 'skip' : track.status === 'silent' ? 'silent' : 'pass'; + if (status === track.status && scenarios.every((scenario, index) => + scenario.overall_passed === track.scenarios[index].overall_passed)) return track; + return { ...track, scenarios, status }; + }); + if (tracks.every((track, index) => track === result.tracks[index])) return result; + const tracksPassed = tracks.filter(t => t.status === 'pass' || t.status === 'silent').length; + const tracksFailed = tracks.filter(t => t.status === 'fail').length; + const tracksPartial = tracks.filter(t => t.status === 'partial').length; + const terminalFailure = result.overall_status === 'auth_required' || result.overall_status === 'unreachable'; + const normalized = { + ...result, tracks, + overall_status: terminalFailure ? result.overall_status + : tracksFailed > 0 ? 'failing' : tracksPartial > 0 || tracksPassed === 0 ? 'partial' : 'passing', + summary: { ...result.summary, tracks_passed: tracksPassed, tracks_failed: tracksFailed, + tracks_partial: tracksPartial, tracks_skipped: tracks.filter(t => t.status === 'skip').length }, + } satisfies ComplianceResult; + const context = pinnedStoryboardContexts.get(result); + if (context) pinnedStoryboardContexts.set(normalized, context); + return normalized; } /** @@ -891,7 +866,7 @@ function effectiveRunStatus(result: ComplianceResult): { const hasFixtureUnavailable = result.tracks.some(trackHasFixtureUnavailableSkip); const hasCoverageGapSkip = hasFixtureUnavailable || result.tracks .filter((track: TrackResult) => track.status === 'skip') - .some(trackHasCoverageGapSkip); + .some(track => trackHasCoverageGapSkip(result, track)); const hasGenuineFailure = activeTracks.some((track: TrackResult) => track.status === 'fail') || result.overall_status === 'failing' || result.overall_status === 'auth_required' || @@ -979,49 +954,9 @@ export function deriveStoryboardStatuses( skippedCount: number; firstFailure: FirstFailure | null; } - const branchSkipReasons = new Set([ - 'peer_branch_taken', - 'peer_substituted', - ]); - const cascadeSkipReasons = new Set([ - 'prerequisite_failed', - ]); - const isControllerSkip = (step: { skip_reason?: string; requirement?: string }): boolean => - step.skip_reason === 'missing_test_controller' || - (step.skip_reason === 'requirement_unmet' && step.requirement === 'controller'); - const isBranchSkip = (step: { skip_reason?: string }): boolean => - branchSkipReasons.has(step.skip_reason ?? ''); - const isCascadeSkip = (step: { skip_reason?: string }): boolean => - cascadeSkipReasons.has(step.skip_reason ?? ''); - const isNeutralApplicabilitySkip = ( - step: { - skip_reason?: string; - step?: unknown; - step_id?: unknown; - details?: unknown; - error?: unknown; - warnings?: unknown; - requirement?: unknown; - skip?: { detail?: unknown }; - }, - scenario: string, - ): boolean => { - if ( - step.skip_reason === 'not_applicable' || - step.skip_reason === 'fixture_unavailable' - ) return true; - - // `@adcp/sdk` emits a single synthetic `missing_tool` phase when a - // storyboard's `required_tools` gate is unmet. That means the storyboard is - // out of scope for this agent, not that a claimed production step failed. - if (isSyntheticRequiredToolsMissingSkip(step, scenario)) return true; - if (isExplicitRequiresToolMissingSkip(step)) return true; - if (isRunnerApplicabilitySkip(step, scenario)) return true; - - return false; - }; const perStoryboard = new Map(); - // Storyboard IDs can contain slashes; the last component is the phase ID. + // Storyboard IDs can contain slashes (e.g. media_buy_seller/available_actions). + // The last component is the phase ID, matching the SDK projection. const tracks = result.tracks ?? []; for (const track of tracks) { @@ -1059,43 +994,19 @@ export function deriveStoryboardStatuses( } continue; } - const phaseApplicabilityReasons = new Set(); - const phaseNonApplicabilityReasons = new Set(); for (const step of s.steps) { - if (step.skipped) { - if (step.skip_reason === 'fixture_unavailable') { - agg.fixtureUnavailable = true; - } - if (isBranchSkip(step)) { - continue; - } - if (isControllerSkip(step)) { - agg.controllerSkipped++; - phaseApplicabilityReasons.add('missing_test_controller'); - continue; - } - if (isNeutralApplicabilitySkip(step, String(s.scenario))) { - if (step.skip_reason) phaseApplicabilityReasons.add(step.skip_reason); - continue; - } - if (isApplicabilityCascade(step, phaseApplicabilityReasons, phaseNonApplicabilityReasons)) { - continue; - } - if (step.skip_reason !== 'prerequisite_failed' && step.skip_reason) { - phaseNonApplicabilityReasons.add(step.skip_reason); - } - if (isCascadeSkip(step)) { - agg.skippedCount++; - } else { - agg.failureCount++; - if (!agg.firstFailure) agg.firstFailure = summarizeStoryboardStepFailure(step); - } - agg.stepsTotal++; + const disposition = classifyRunStep(result, String(s.scenario), step); + if (disposition === 'not_applicable') continue; + if (disposition === 'setup_gap') { + agg.controllerSkipped++; + if (step.skip_reason === 'fixture_unavailable') agg.fixtureUnavailable = true; continue; } agg.stepsTotal++; - if (step.passed) { + if (disposition === 'passed') { agg.stepsPassed++; + } else if (disposition === 'dependency_failed') { + agg.skippedCount++; } else { agg.failureCount++; if (!agg.firstFailure) agg.firstFailure = summarizeStoryboardStepFailure(step); @@ -1104,6 +1015,17 @@ export function deriveStoryboardStatuses( } } + for (const observation of result.observations ?? []) { + if (observation.source?.code !== 'storyboard-assertion-failed') continue; + const id = 'storyboard_id' in observation.source ? observation.source.storyboard_id : undefined; + if (!id || tracks.some(track => track.scenarios.some(scenario => scenario.scenario === `${id}/storyboard`))) continue; + const agg = perStoryboard.get(id); + if (agg) { + agg.failureCount++; + agg.stepsTotal++; + } + } + // Decide which storyboard ids to emit entries for. const hasExplicitIds = !!storyboardIds && storyboardIds.length > 0; const toEmit = hasExplicitIds ? storyboardIds! : Array.from(perStoryboard.keys()); @@ -1122,7 +1044,7 @@ export function deriveStoryboardStatuses( // Fixture availability is a whole-storyboard preflight disposition. The // producer step that captured the format may already have passed, but that // partial execution is not evidence that the seller behavior was tested. - if (agg.fixtureUnavailable) { + if (agg.fixtureUnavailable && agg.failureCount === 0 && agg.skippedCount === 0) { entries.push({ storyboard_id: sbId, status: 'untested', @@ -1422,20 +1344,21 @@ export function complianceResultToDbInput( triggeredBy: TriggeredBy = 'manual', storyboardIds?: string[], ): RecordComplianceRunInput { + const gradedResult = normalizeGradedTracks(result); const selectedStoryboardIds = storyboardIds?.length ? storyboardIds : result.bundle_results?.length ? [...new Set(result.bundle_results.flatMap(bundle => bundle.storyboard_ids))].sort() : undefined; - const tracksJson: TrackSummaryEntry[] = result.tracks.map((t: TrackResult) => ({ + const tracksJson: TrackSummaryEntry[] = gradedResult.tracks.map((t: TrackResult) => ({ track: t.track, status: t.status, scenario_count: t.scenarios.length, passed_count: t.scenarios.filter((s: { overall_passed: boolean }) => s.overall_passed).length, duration_ms: t.duration_ms, - has_coverage_gap_skip: trackHasCoverageGapSkip(t), + has_coverage_gap_skip: trackHasCoverageGapSkip(gradedResult, t), })); - const { overall_status, tracks_passed, tracks_failed, tracks_partial } = effectiveRunStatus(result); + const { overall_status, tracks_passed, tracks_failed, tracks_partial } = effectiveRunStatus(gradedResult); const resultWithCompatibleNotices = result as unknown as { notices?: NoticeEntry[] | null; summary?: { notices?: NoticeEntry[] | null }; @@ -1460,11 +1383,14 @@ export function complianceResultToDbInput( tracks_json: tracksJson, tracks_passed, tracks_failed, - tracks_skipped: result.summary.tracks_skipped, + tracks_skipped: gradedResult.summary.tracks_skipped, tracks_partial, agent_profile_json: result.agent_profile, observations_json: result.observations, triggered_by: triggeredBy, + provenance_json: redactForDiagnostics( + (result as ComplianceResult & { hosted_provenance?: ComplianceRunProvenance }).hosted_provenance ?? complianceRunProvenance(result), + ) as ComplianceRunProvenance, completeness: result.completeness ?? 'complete', is_authoritative: result.completeness !== 'timed_out' && result.overall_status !== 'auth_required' && !storyboardIds?.length, storyboard_statuses: deriveStoryboardStatuses(result, selectedStoryboardIds), @@ -1518,7 +1444,7 @@ const SENSITIVE_DIAGNOSTIC_KEY_PATTERN = /(authorization|token|secret|password|c const SENSITIVE_DIAGNOSTIC_VALUE_PATTERN = /\b(?:sk_(?:live|test)_[A-Za-z0-9_]{12,}|gh[pousr]_[A-Za-z0-9_]{20,}|xox[baprs]-[A-Za-z0-9-]{12,}|eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b/; const SENSITIVE_DIAGNOSTIC_TEXT_PATTERN = /(?:\bbearer\s+\S+|\b(?:authorization|cookie|set-cookie|session|api[_ -]?key|access[_ -]?token|refresh[_ -]?token|secret|password|credential)\b\s*[:=]\s*\S+)/i; -function redactForDiagnostics(value: unknown, depth = 0): unknown { +export function redactForDiagnostics(value: unknown, depth = 0): unknown { if (value === undefined || value === null) return value; if (typeof value === 'string') { return SENSITIVE_DIAGNOSTIC_VALUE_PATTERN.test(value) || SENSITIVE_DIAGNOSTIC_TEXT_PATTERN.test(value) @@ -1635,8 +1561,8 @@ export function extractFailingStepDiagnostics(result: ComplianceResult): StepDia response_jsonb: capDiagnosticPayload(responsePayload), extraction_path: extraction?.path, extraction_note: extraction?.note, - error_text: redactDiagnosticText(step.error), - adcp_error_jsonb: capDiagnosticPayload(step.adcp_error), + error_text: redactDiagnosticText(step.error ?? matchedFailure?.error), + adcp_error_jsonb: capDiagnosticPayload(step.adcp_error ?? matchedFailure?.adcp_error), failed_validations_jsonb: failedValidations && failedValidations.length > 0 ? capDiagnosticPayload(failedValidations.map(stripValidationRequestResponse)) : undefined, @@ -1831,15 +1757,20 @@ function remainingFailures(lookup: FailureLookups): ComplianceFailureSummary[] { return out; } -function phaseIdForFailure(_result: ComplianceResult, failure: ComplianceFailureSummary): string | undefined { - const storyboard = getStoryboard(failure.storyboard_id); - const byId = storyboard?.phases.find(phase => - phase.steps.some(step => step.id === failure.step_id), - )?.id; - if (byId) return byId; - return storyboard?.phases.find(phase => - phase.steps.some(step => step.title === failure.step_title && step.task === failure.task), - )?.id; +function phaseIdForFailure(result: ComplianceResult, failure: ComplianceFailureSummary): string | undefined { + if (!result.adcp_version) return undefined; + try { + const storyboard = pinnedStoryboard(result, failure.storyboard_id); + const byId = storyboard?.phases.find(phase => + phase.steps.some(step => step.id === failure.step_id), + )?.id; + if (byId) return byId; + return storyboard?.phases.find(phase => + phase.steps.some(step => step.title === failure.step_title && step.task === failure.task), + )?.id; + } catch { + return undefined; + } } function failedValidationsForStep( diff --git a/server/src/compliance/run-provenance.ts b/server/src/compliance/run-provenance.ts new file mode 100644 index 00000000000..0a1d5fbd1da --- /dev/null +++ b/server/src/compliance/run-provenance.ts @@ -0,0 +1,32 @@ +import { LIBRARY_VERSION } from '@adcp/sdk'; +import type { ComplianceResult, ComplyOptions } from '@adcp/sdk/testing'; + +export interface ComplianceRunProvenance { + grading_policy_version: string; + compliance_bundle_version: string | null; + sdk_version: string; + agent_build_version: string | null; + agent_library_version: string | null; + test_session_id: string | null; + timeout_ms: number | null; + storyboard_start_offset: number | null; + auth_type: string | null; +} + +export function complianceRunProvenance( + result: Pick, + options?: ComplyOptions, +): ComplianceRunProvenance { + return { + grading_policy_version: 'hosted-compliance-v1', + compliance_bundle_version: result.adcp_version ?? null, + sdk_version: LIBRARY_VERSION, + // Agent-reported provenance; absent optional fields remain unknown. + agent_build_version: result.agent_profile?.adcp_build_version ?? null, + agent_library_version: result.agent_profile?.library_version ?? null, + test_session_id: options?.test_session_id ?? null, + timeout_ms: options?.timeout_ms ?? null, + storyboard_start_offset: options?.storyboard_start_offset ?? null, + auth_type: options?.auth?.type ?? null, + }; +} diff --git a/server/src/compliance/step-disposition.ts b/server/src/compliance/step-disposition.ts new file mode 100644 index 00000000000..8ee57c90750 --- /dev/null +++ b/server/src/compliance/step-disposition.ts @@ -0,0 +1,55 @@ +/** Hosted grading policy over SDK output; never derive a verdict from diagnostic prose. */ +export type StepDisposition = 'passed' | 'failed' | 'not_applicable' | 'setup_gap' | 'dependency_failed'; + +type GradedStep = { + passed?: boolean; + skipped?: boolean; + task?: string; + step_id?: unknown; + skip_reason?: string; + requirement?: unknown; + skip?: { reason?: string; requirement?: unknown }; +}; + +// @adcp/sdk 14.0.0-rc.33 projects detailed reasons but omits canonical skip.reason. +// Keep this exact compatibility list aligned with its DETAILED_SKIP_TO_CANONICAL. +const NOT_APPLICABLE = new Set([ + 'not_applicable', 'peer_branch_taken', 'peer_substituted', + 'probe_skipped', 'not_in_only_vectors', 'grader_skipped', + 'capability_profile_mismatch', 'transport_ungradable', 'mcp_mode_flattens_url_edges', + 'oauth_not_advertised', 'rate_limit_not_triggered', 'force_scenario_unsupported', + 'fixture_seed_unsupported', 'fixture_unsatisfied', 'capability_unsupported', + 'capability_prerequisite_unavailable', +]); +const SETUP_GAPS = new Set([ + 'missing_test_controller', 'fixture_unavailable', 'unsatisfied_contract', + 'rate_abuse_opt_out', 'live_side_effect_opt_in_required', + 'operator_skip', 'controller_seeding_failed', +]); +export function classifyComplianceStep(step: GradedStep, scenario = ''): StepDisposition { + // The synthetic seeding phase evaluates harness setup, not production behavior. + // Retain its failed wire evidence separately for the owner/operator. + if (scenario.endsWith('/__controller_seeding__') && step.task === 'comply_test_controller') { + return step.passed && !step.skipped ? 'not_applicable' : 'setup_gap'; + } + if (!step.skipped) return step.passed ? 'passed' : 'failed'; + const reason = step.skip?.reason ?? step.skip_reason; + if (reason === 'missing_tool') { + // rc.33 omits step_id in the synthetic projection. If a producer supplies + // an identity, require it to agree with the reserved gate phase. + const syntheticGate = scenario.endsWith('/missing_tool') && + (step.step_id === undefined || step.step_id === 'missing_tool'); + return syntheticGate ? 'not_applicable' : 'failed'; + } + if (NOT_APPLICABLE.has(reason ?? '')) return 'not_applicable'; + // The detailed seeding reason is more specific than canonical prerequisite_failed. + if (SETUP_GAPS.has(reason ?? '') || step.skip_reason === 'controller_seeding_failed') return 'setup_gap'; + const requirement = step.skip?.requirement ?? step.requirement; + if (reason === 'requirement_unmet' && typeof requirement === 'string' && requirement.trim()) { + return 'setup_gap'; + } + // rc.33 drops causal IDs from ordinary prerequisite failures. A nearby neutral + // skip is not proof: a passed producer may still have omitted required state. + if (reason === 'prerequisite_failed') return 'dependency_failed'; + return 'failed'; +} diff --git a/server/src/db/compliance-db.ts b/server/src/db/compliance-db.ts index d48e9bb47ff..bc69ea2d420 100644 --- a/server/src/db/compliance-db.ts +++ b/server/src/db/compliance-db.ts @@ -1,3 +1,4 @@ +import type { ComplianceRunProvenance } from '../compliance/run-provenance.js'; import { isAuthoritativeComplianceRun, type RunCompleteness } from '../compliance/run-publication.js'; import { query, getClient } from './client.js'; import { decrypt as decryptToken } from './encryption.js'; @@ -131,6 +132,7 @@ export interface ComplianceRun { completeness: RunCompleteness; is_authoritative: boolean; storyboard_statuses_json: StoryboardStatusEntry[] | null; + provenance_json: ComplianceRunProvenance | null; notices_json: NoticeEntry[] | null; } @@ -165,6 +167,7 @@ export interface AgentComplianceStatus { last_triggered_by: TriggeredBy | null; /** tracks_json from the most recent non-dry-run, used for current-run UI details */ track_details_json?: TrackSummaryEntry[] | null; + provenance_json?: ComplianceRunProvenance | null; } export interface ComplianceStatusWithStoryboardCounts { @@ -294,6 +297,7 @@ export interface RecordComplianceRunInput { dry_run?: boolean; completeness?: RunCompleteness; is_authoritative?: boolean; + provenance_json?: ComplianceRunProvenance | null; /** Durable refresh operation that produced this run; makes lease recovery idempotent. */ refresh_operation_id?: string | null; /** Lease token paired with refresh_operation_id for fenced persistence. */ @@ -617,8 +621,8 @@ export class ComplianceDatabase { total_duration_ms, tracks_json, tracks_passed, tracks_failed, tracks_skipped, tracks_partial, agent_profile_json, observations_json, triggered_by, triggered_org_id, dry_run, - notices_json, refresh_operation_id, completeness, is_authoritative, storyboard_statuses_json - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) + notices_json, refresh_operation_id, completeness, is_authoritative, storyboard_statuses_json, provenance_json + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23) ON CONFLICT (refresh_operation_id) DO NOTHING RETURNING *`, [ @@ -644,6 +648,7 @@ export class ComplianceDatabase { input.completeness ?? 'complete', authoritative, JSON.stringify(input.storyboard_statuses ?? []), + input.provenance_json ? JSON.stringify(input.provenance_json) : null, ], ); let run = runResult.rows[0] as ComplianceRun | undefined; @@ -1000,11 +1005,11 @@ export class ComplianceDatabase { `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage, r.id AS last_run_id, r.triggered_by AS last_triggered_by, - r.tracks_json AS track_details_json + r.tracks_json AS track_details_json, r.provenance_json FROM agent_compliance_status s LEFT JOIN agent_registry_metadata m ON m.agent_url = s.agent_url LEFT JOIN LATERAL ( - SELECT id, triggered_by, tracks_json FROM agent_compliance_runs + SELECT id, triggered_by, tracks_json, provenance_json FROM agent_compliance_runs WHERE agent_url = s.agent_url AND dry_run = false AND is_authoritative = true ORDER BY tested_at DESC LIMIT 1 ) r ON true @@ -1019,13 +1024,13 @@ export class ComplianceDatabase { `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage, r.id AS last_run_id, r.triggered_by AS last_triggered_by, - r.tracks_json AS track_details_json, + r.tracks_json AS track_details_json, r.provenance_json, COALESCE(sb_counts.passing, 0)::int AS storyboards_passing, COALESCE(sb_counts.total, 0)::int AS storyboards_total FROM agent_compliance_status s LEFT JOIN agent_registry_metadata m ON m.agent_url = s.agent_url LEFT JOIN LATERAL ( - SELECT id, triggered_by, tracks_json FROM agent_compliance_runs + SELECT id, triggered_by, tracks_json, provenance_json FROM agent_compliance_runs WHERE agent_url = s.agent_url AND dry_run = false AND is_authoritative = true ORDER BY tested_at DESC LIMIT 1 ) r ON true @@ -1080,11 +1085,11 @@ export class ComplianceDatabase { `SELECT s.*, COALESCE(m.lifecycle_stage, 'production') AS lifecycle_stage, r.id AS last_run_id, r.triggered_by AS last_triggered_by, - r.tracks_json AS track_details_json + r.tracks_json AS track_details_json, r.provenance_json FROM agent_compliance_status s LEFT JOIN agent_registry_metadata m ON m.agent_url = s.agent_url LEFT JOIN LATERAL ( - SELECT id, triggered_by, tracks_json FROM agent_compliance_runs + SELECT id, triggered_by, tracks_json, provenance_json FROM agent_compliance_runs WHERE agent_url = s.agent_url AND dry_run = false AND is_authoritative = true ORDER BY tested_at DESC LIMIT 1 ) r ON true diff --git a/server/src/db/migrations/590_compliance_run_provenance.sql b/server/src/db/migrations/590_compliance_run_provenance.sql new file mode 100644 index 00000000000..cf49349aa02 --- /dev/null +++ b/server/src/db/migrations/590_compliance_run_provenance.sql @@ -0,0 +1 @@ +ALTER TABLE agent_compliance_runs ADD COLUMN provenance_json JSONB; diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 6cc19c42935..06d2b7396ae 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -43,6 +43,7 @@ import { import { comply, complianceResultToDbInput, + redactForDiagnostics, isNonExecutableCoverageGapScenario, classifyCapabilityResolutionError, presentCapabilityResolutionError, @@ -134,6 +135,7 @@ import { RegistryMetadataSchema, MonitoringSettingsSchema, ComplianceRunSchema, + ComplianceRunProvenanceSchema, ComplianceStepDiagnosticSchema, OutboundRequestSchema, AgentAuthStatusSchema, @@ -709,7 +711,7 @@ interface PublicComplianceObservation { message: string; } -function toPublicComplianceObservation(obs: unknown): PublicComplianceObservation | null { +function toPublicComplianceObservation(obs: unknown, includeDiagnostics = false): PublicComplianceObservation | null { if (!obs || typeof obs !== "object") return null; const record = obs as Record; if ( @@ -722,10 +724,18 @@ function toPublicComplianceObservation(obs: unknown): PublicComplianceObservatio return { category: record.category, severity: record.severity, - message: record.message, + message: record.severity === 'error' && !includeDiagnostics + ? 'Compliance failure recorded. Details are available to the owner or operator.' + : String(redactForDiagnostics(record.message)), }; } +function publicComplianceHeadline(status: string, headline: string | null): string | null { + if (!headline) return null; + return status === 'passing' ? String(redactForDiagnostics(headline)) + : 'Compliance assessment recorded. Details are available to the owner or operator.'; +} + /** Strip protocol, path, query, and fragment from a URL to extract the domain. */ function extractDomain(raw: string): string { let d = raw.replace(/^https?:\/\//, ""); @@ -4038,6 +4048,13 @@ registry.registerPath({ schema: z.object({ agent_url: z.string(), run_id: z.string().nullable(), + completeness: z.enum(['complete', 'timed_out', 'not_completed']).nullable(), + is_authoritative: z.boolean(), + provenance: ComplianceRunProvenanceSchema.nullable(), + storyboard_statuses: z.array(z.any()), + observations: z.array(z.unknown()), + diagnostics_visibility: z.literal('owner_or_operator'), + diagnostics_note: z.string(), count: z.number().int(), diagnostics: z.array(ComplianceStepDiagnosticSchema), }), @@ -4140,6 +4157,7 @@ registry.registerPath({ ran: z.boolean().openapi({ description: "True if a compliance run was recorded. Check completeness and is_authoritative before treating its results as the public grade." }), completeness: z.enum(['complete', 'timed_out', 'not_completed']).optional(), is_authoritative: z.boolean().optional().openapi({ description: "Only authoritative runs update the public grade and badge state. Incomplete runs preserve the previous complete assessment." }), + provenance: ComplianceRunProvenanceSchema.nullable().optional(), run_id: z.string().optional().openapi({ description: "Compliance run id written by this refresh. Use with /compliance/diagnostics?run_id=... to inspect failing-step wire evidence." }), test_session_id: z.string().optional().openapi({ description: "Fresh test session id used for the compliance run. Useful when matching seller-side logs to the refresh." }), requested_compliance_target: z.string().optional().openapi({ description: "Requested compliance target before alias resolution, e.g. 3.1, 3.0, 3.1-rc, or 3.1-beta. Present when `ran` is true." }), @@ -4942,6 +4960,7 @@ registry.registerPath({ run_id: z.string().openapi({ description: "Compliance run id written by this owner-triggered storyboard run." }), completeness: z.enum(['complete', 'timed_out', 'not_completed']), is_authoritative: z.boolean(), + provenance: ComplianceRunProvenanceSchema.nullable(), storyboard_status: StoryboardRunStatusResponseSchema.openapi({ description: "Persisted storyboard verdict for this run, using the same executable-step semantics as agent_storyboard_status." }), phases: z.any(), summary: z.any(), @@ -6745,9 +6764,10 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { lifecycle_stage: cs.lifecycle_stage, tracks: cs.tracks_summary_json || {}, track_details: cs.track_details_json || [], + provenance: cs.provenance_json ?? null, streak_days: cs.streak_days, last_checked_at: cs.last_checked_at?.toISOString() || null, - headline: cs.headline, + headline: publicComplianceHeadline(cs.status, cs.headline), monitoring_paused: meta?.monitoring_paused ?? false, check_interval_hours: meta?.check_interval_hours ?? 12, verified: agentBadges.length > 0, @@ -6855,11 +6875,9 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { // observations (best-practice warnings, suggestions, etc.). Do not merge // observations across runs; a fixed field on the wire must clear the // advisory as soon as the latest run stops emitting it. - let observations: PublicComplianceObservation[] = []; + let rawObservations: unknown[] = []; try { - observations = (await complianceDb.getLatestObservations(agentUrl)) - .map(toPublicComplianceObservation) - .filter((obs): obs is PublicComplianceObservation => obs !== null); + rawObservations = await complianceDb.getLatestObservations(agentUrl); } catch (err) { logger.warn({ err, agentUrl }, "Latest observations query failed"); } @@ -6924,8 +6942,11 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } const encodedUrl = encodeURIComponent(agentUrl); + const includeDiagnostics = ownerMembership.is_owner || isStaticAdminRequest(req); + const observations = rawObservations.map(obs => toPublicComplianceObservation(obs, includeDiagnostics)) + .filter((obs): obs is PublicComplianceObservation => obs !== null); const serializedStoryboardStatuses = storyboardStatuses.map(s => - serializeStoryboardStatus(s, { includeDiagnostics: ownerMembership.is_owner }), + serializeStoryboardStatus(s, { includeDiagnostics }), ); res.json({ @@ -6938,11 +6959,12 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { badge_requalification_required: metadata?.badge_requalification_required ?? false, tracks: status.tracks_summary_json || {}, track_details: status.track_details_json || [], + provenance: status.provenance_json ?? null, streak_days: status.streak_days, last_checked_at: status.last_checked_at?.toISOString() || null, last_passed_at: status.last_passed_at?.toISOString() || null, last_failed_at: status.last_failed_at?.toISOString() || null, - headline: status.headline, + headline: includeDiagnostics ? redactForDiagnostics(status.headline) : publicComplianceHeadline(status.status, status.headline), status_changed_at: status.status_changed_at?.toISOString() || null, storyboards_passing: sbCounts.passing, storyboards_total: sbCounts.total, @@ -7027,10 +7049,13 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { agent_url: agentUrl, runs: history.map(run => ({ id: run.id, + completeness: run.completeness, + is_authoritative: run.is_authoritative, + provenance: run.provenance_json ?? null, requested_compliance_target: run.requested_compliance_target ?? null, adcp_version: run.adcp_version ?? null, overall_status: run.overall_status, - headline: run.headline, + headline: publicComplianceHeadline(run.overall_status, run.headline), tracks_passed: run.tracks_passed, tracks_failed: run.tracks_failed, tracks_skipped: run.tracks_skipped, @@ -7941,6 +7966,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { overall_status?: string; completeness?: string; is_authoritative?: boolean; + provenance?: unknown; storyboards_passing?: number; storyboards_total?: number; run_id?: string; @@ -7961,6 +7987,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { overall_status: run.overall_status, completeness: run.completeness, is_authoritative: isAuthoritativeComplianceRun(run), + provenance: run.provenance_json ?? null, storyboards_passing: storyboardStatuses.filter(status => status.status === 'passing').length, storyboards_total: storyboardStatuses.length, observations_count: Array.isArray(run.observations_json) ? run.observations_json.length : 0, @@ -8082,6 +8109,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { overall_status: run.overall_status, completeness: run.completeness, is_authoritative: isAuthoritativeComplianceRun(run), + provenance: run.provenance_json ?? null, storyboards_passing: passing, storyboards_total: storyboardStatuses.length, observations_count: Array.isArray(run.observations_json) ? run.observations_json.length : 0, @@ -8428,11 +8456,19 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { limit = Math.min(Math.floor(parsed), 1000); } - const rows = await complianceDb.getStepDiagnostics(agentUrl, { runId: runIdRaw, limit }); + const run = await complianceDb.getComplianceRun(agentUrl, runIdRaw); + const rows = run ? await complianceDb.getStepDiagnostics(agentUrl, { runId: run.id, limit }) : []; res.json({ agent_url: agentUrl, - run_id: runIdRaw ?? (rows[0]?.run_id ?? null), + run_id: run?.id ?? null, + completeness: run?.completeness ?? null, + is_authoritative: run?.is_authoritative ?? false, + provenance: run?.provenance_json ?? null, + storyboard_statuses: run?.storyboard_statuses_json ?? [], + observations: redactForDiagnostics(run?.observations_json ?? []), + diagnostics_visibility: 'owner_or_operator', + diagnostics_note: 'Credentials are redacted. Missing wire fields were not captured by the SDK; an absent agent build version is unknown.', count: rows.length, diagnostics: rows, }); @@ -9166,6 +9202,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { const complyOptions = { timeout_ms: 90_000, storyboards: [req.params.storyboardId], + test_session_id: randomUUID(), ...(sdkAuth && { auth: sdkAuth }), }; const seededSupportedVersions = await complianceDb.getLastKnownSupportedVersions(agentUrl); @@ -9293,6 +9330,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { run_id: run.id, completeness: run.completeness, is_authoritative: isAuthoritativeComplianceRun(run), + provenance: run.provenance_json ?? null, storyboard_status: serializedStoryboardStatus, phases: annotatedPhases, summary: complyResult.summary, diff --git a/server/src/schemas/registry.ts b/server/src/schemas/registry.ts index 35f93cb4c01..f7cb1722194 100644 --- a/server/src/schemas/registry.ts +++ b/server/src/schemas/registry.ts @@ -770,8 +770,21 @@ export const PropertyRegistryItemSchema = z }) .openapi("PropertyRegistryItem"); +export const ComplianceRunProvenanceSchema = z.object({ + grading_policy_version: z.string(), + compliance_bundle_version: z.string().nullable(), + sdk_version: z.string(), + agent_build_version: z.string().nullable().openapi({ description: 'Agent-reported adcp.build_version; null when not supplied. Never inferred from the endpoint.' }), + agent_library_version: z.string().nullable(), + test_session_id: z.string().nullable(), + timeout_ms: z.number().nullable(), + storyboard_start_offset: z.number().nullable(), + auth_type: z.string().nullable(), +}).openapi('ComplianceRunProvenance'); + export const AgentComplianceSchema = z .object({ + provenance: ComplianceRunProvenanceSchema.nullable().optional(), status: z.enum(["passing", "degraded", "failing", "unknown"]), requested_compliance_target: z.string().nullable().optional().openapi({ description: "Requested compliance target before alias resolution, e.g. 3.0 or 3.1-beta." }), adcp_version: z.string().nullable().optional().openapi({ description: "Concrete AdCP compliance bundle version used for the latest run, e.g. 3.0.12." }), @@ -817,6 +830,7 @@ export const VerificationBadgeSchema = z export const AgentComplianceDetailSchema = z .object({ + provenance: ComplianceRunProvenanceSchema.nullable().optional(), agent_url: z.string(), requested_compliance_target: z.string().nullable().optional().openapi({ description: "Requested compliance target before alias resolution, e.g. 3.0 or 3.1-beta. Null for legacy rows before target recording." }), adcp_version: z.string().nullable().optional().openapi({ description: "Concrete AdCP compliance bundle version used for the latest run, e.g. 3.0.12. Null for legacy rows before version recording." }), @@ -1325,7 +1339,10 @@ export const MonitoringSettingsSchema = z export const ComplianceRunSchema = z .object({ + provenance: ComplianceRunProvenanceSchema.nullable().optional(), id: z.string(), + completeness: z.enum(['complete', 'timed_out', 'not_completed']).optional(), + is_authoritative: z.boolean().optional(), requested_compliance_target: z.string().nullable().optional(), adcp_version: z.string().nullable().optional(), overall_status: z.string(), diff --git a/server/src/types.ts b/server/src/types.ts index d4bbdeea296..f9fef3edb8a 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -1171,6 +1171,7 @@ export interface CommitteeDocumentActivity { // Agent Compliance Types export interface AgentCompliance { + provenance?: import('./compliance/run-provenance.js').ComplianceRunProvenance | null; status: 'passing' | 'degraded' | 'failing' | 'unknown'; requested_compliance_target?: string | null; adcp_version?: string | null; diff --git a/server/tests/integration/registry-api-agent-refresh.test.ts b/server/tests/integration/registry-api-agent-refresh.test.ts index 517d0fc98e4..3ed7c851e9d 100644 --- a/server/tests/integration/registry-api-agent-refresh.test.ts +++ b/server/tests/integration/registry-api-agent-refresh.test.ts @@ -21,6 +21,8 @@ import { AAO_UA_COMPLIANCE } from '../../src/config/user-agents.js'; import { HOSTED_FULL_COMPLIANCE_TIMEOUT_MS } from '../../src/services/hosted-compliance-version.js'; import { ComplianceRefreshRequestsDatabase } from '../../src/db/compliance-refresh-requests-db.js'; import { ComplianceDatabase } from '../../src/db/compliance-db.js'; +import { complianceRunProvenance } from '../../src/compliance/run-provenance.js'; +import type { ComplianceResult, ComplyOptions } from '@adcp/sdk/testing'; vi.hoisted(() => { process.env.WORKOS_API_KEY ??= 'sk_test_registry_refresh'; @@ -61,6 +63,8 @@ const ALL_OWNED_URLS = [ ownedAgentUrl('badge-retry'), ownedAgentUrl('badge-retry-exhausted'), ownedAgentUrl('legacy-timeout'), + ownedAgentUrl('targeted-complete'), + ownedAgentUrl('targeted-timed_out'), ]; // Toggle which user the auth middleware stamps onto the request. Tests @@ -325,6 +329,26 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { const url = (agentUrl: string) => `/api/registry/agents/${encodeURIComponent(agentUrl)}/refresh`; + it.each(['complete', 'timed_out'] as const)('reports targeted %s runs as audit-only with provenance and no diagnostics', async completeness => { + const agentUrl = ownedAgentUrl(`targeted-${completeness}`); + complyMock.mockImplementation(async (_url: string, options: ComplyOptions) => { + const result = { ...makeComplianceResult(), completeness, adcp_version: '3.0.22' } as unknown as ComplianceResult; + return { ...result, hosted_provenance: complianceRunProvenance(result, options) }; + }); + const res = await request(app).post(`/api/registry/agents/${encodeURIComponent(agentUrl)}/storyboard/media_buy_seller/run`).send(); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ completeness, is_authoritative: false, badge_eligible: false, + badge_eligible_adcp_versions: [], diagnostics: [], + provenance: { sdk_version: '14.0.0-rc.33', test_session_id: expect.any(String), agent_build_version: null }, + }); + expect(complyMock.mock.calls[0][1].test_session_id).toBe(res.body.provenance.test_session_id); + const db = new ComplianceDatabase(); + expect(await db.getComplianceStatus(agentUrl)).toBeNull(); + expect(await db.getBadgesForAgent(agentUrl)).toEqual([]); + expect((await db.getComplianceRun(agentUrl, res.body.run_id))?.provenance_json?.test_session_id) + .toBe(res.body.provenance.test_session_id); + }); + it('owner can refresh and gets the snapshot back', async () => { const agentUrl = ownedAgentUrl('owner'); const res = await request(app).post(url(agentUrl)).send(); @@ -731,7 +755,9 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { expect(materialized.rowCount).toBe(0); const history = await request(app).get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/compliance/history`); expect(history.body.runs).toEqual([]); - + const diagnostics = await request(app).get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/compliance/diagnostics`); + expect(diagnostics.body).toMatchObject({ completeness: 'timed_out', is_authoritative: false, + diagnostics_visibility: 'owner_or_operator', provenance: { sdk_version: '14.0.0-rc.33', agent_build_version: null } }); } } finally { revokeBadges.mockRestore(); diff --git a/server/tests/integration/registry-api-compliance-verdict-source.test.ts b/server/tests/integration/registry-api-compliance-verdict-source.test.ts index e23b3dbeabf..79b89a82fc9 100644 --- a/server/tests/integration/registry-api-compliance-verdict-source.test.ts +++ b/server/tests/integration/registry-api-compliance-verdict-source.test.ts @@ -27,6 +27,8 @@ import request from 'supertest'; import type { Pool } from 'pg'; import { HTTPServer } from '../../src/http.js'; import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; +import { ComplianceDatabase } from '../../src/db/compliance-db.js'; +import { complianceResultToDbInput, type ComplianceResult } from '../../src/addie/services/compliance-testing.js'; import { runMigrations } from '../../src/db/migrate.js'; vi.hoisted(() => { @@ -453,6 +455,81 @@ describe('GET /api/registry/agents/:encodedUrl/compliance — owner-scope gate ( ])); }); + it('keeps partial-run provenance and redacted failures visible only to the owner/operator', async () => { + const db = new ComplianceDatabase(); + const partial = complianceResultToDbInput({ + completeness: 'timed_out', adcp_version: '3.1.20', overall_status: 'failing', + agent_profile: { tools: [], adcp_build_version: 'build-immutable-42', library_version: 'seller-sdk-2' }, + summary: { headline: 'Partial', tracks_passed: 0, tracks_failed: 1, tracks_partial: 0, tracks_skipped: 0 }, + tracks: [{ track: 'core', status: 'fail', duration_ms: 1, scenarios: [{ + scenario: 'debug_storyboard/check', overall_passed: false, steps: [{ + step_id: 'debug_step', step: 'Read products', task: 'get_products', passed: false, + error: 'Expected products array', observation_data: { api_key: 'fixture-secret-value', products: [] }, + }], + }] }], observations: [], total_duration_ms: 1, + } as unknown as ComplianceResult, AGENT_URL, 'production', 'owner_test'); + const { run } = await db.recordComplianceRun({ ...partial, dry_run: false }); + try { + for (const user of [OWNER_USER_ID, STATIC_ADMIN_USER_ID]) { + currentUserId = user; + const response = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance/diagnostics?run_id=${run.id}`); + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ run_id: run.id, completeness: 'timed_out', is_authoritative: false, + provenance: { compliance_bundle_version: '3.1.20', sdk_version: '14.0.0-rc.33', agent_build_version: 'build-immutable-42', agent_library_version: 'seller-sdk-2' }, + diagnostics_visibility: 'owner_or_operator' }); + expect(response.body.diagnostics[0].error_text).toBe('Expected products array'); + expect(JSON.stringify(response.body)).not.toContain('fixture-secret-value'); + } + for (const user of [null, CROSS_ORG_USER_ID]) { + currentUserId = user; + const card = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance`); + expect(card.body.storyboard_statuses[0].first_failure_message).toBeNull(); + expect(card.body.status).toBe('passing'); + const history = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance/history`); + expect(history.body.runs.some((entry: { id: string }) => entry.id === run.id)).toBe(false); + const debug = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance/diagnostics?run_id=${run.id}`); + expect(debug.status).toBe(user === null ? 401 : 403); + } + } finally { + await pool.query('DELETE FROM agent_compliance_step_diagnostics WHERE run_id = $1', [run.id]); + await pool.query('DELETE FROM agent_compliance_runs WHERE id = $1', [run.id]); + } + }); + + it('keeps observation-only diagnostics useful to owners while redacting public failures and legacy headlines', async () => { + const db = new ComplianceDatabase(); + const previous = (await pool.query('SELECT * FROM agent_compliance_runs WHERE id = $1', [complianceRunId])).rows[0]; + const observations = [{ category: 'setup', severity: 'error', message: 'Controller fixture setup unavailable', evidence: { api_key: 'fixture-private-key' } }]; + await pool.query('UPDATE agent_compliance_runs SET observations_json = $2, headline = $3, overall_status = $4 WHERE id = $1', + [complianceRunId, JSON.stringify(observations), 'Failure: api_key=fixture-private-key', 'failing']); + const { run } = await db.recordComplianceRun({ agent_url: AGENT_URL, lifecycle_stage: 'production', + overall_status: 'failing', tracks_json: [], completeness: 'not_completed', is_authoritative: false, + tracks_passed: 0, tracks_failed: 0, tracks_partial: 0, tracks_skipped: 0, + observations_json: observations, dry_run: false }); + try { + for (const user of [OWNER_USER_ID, STATIC_ADMIN_USER_ID]) { + currentUserId = user; + const card = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance`); + expect(card.body.observations[0].message).toBe('Controller fixture setup unavailable'); + const debug = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance/diagnostics?run_id=${run.id}`); + expect(debug.body).toMatchObject({ run_id: run.id, completeness: 'not_completed', count: 0, diagnostics: [] }); + expect(debug.body.observations[0].message).toBe('Controller fixture setup unavailable'); + expect(JSON.stringify(debug.body)).not.toContain('fixture-private-key'); + expect(JSON.stringify(card.body)).not.toContain('fixture-private-key'); + } + currentUserId = null; + const card = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance`); + expect(card.body.observations[0].message).not.toContain('Controller fixture'); + expect(JSON.stringify(card.body)).not.toContain('fixture-private-key'); + const history = await request(app).get(`/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance/history`); + expect(JSON.stringify(history.body)).not.toContain('fixture-private-key'); + } finally { + await pool.query('DELETE FROM agent_compliance_runs WHERE id = $1', [run.id]); + await pool.query('UPDATE agent_compliance_runs SET observations_json = $2, headline = $3, overall_status = $4 WHERE id = $1', + [complianceRunId, JSON.stringify(previous.observations_json), previous.headline, previous.overall_status]); + } + }); + it('static admin API key can read per-step diagnostics for any agent', async () => { currentUserId = STATIC_ADMIN_USER_ID; const res = await request(app) diff --git a/server/tests/unit/compliance-openapi.test.ts b/server/tests/unit/compliance-openapi.test.ts new file mode 100644 index 00000000000..7591370256d --- /dev/null +++ b/server/tests/unit/compliance-openapi.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import YAML from 'yaml'; + +describe('compliance publication OpenAPI contract', () => { + it('exposes publication and provenance on refresh and targeted run responses', () => { + const spec = YAML.parse(readFileSync(new URL('../../../static/openapi/registry.yaml', import.meta.url), 'utf8')); + const response = (path: string) => spec.paths[path].post.responses['200'].content['application/json'].schema.properties; + const refresh = response('/api/registry/agents/{encodedUrl}/refresh').compliance.properties; + const targeted = response('/api/registry/agents/{encodedUrl}/storyboard/{storyboardId}/run'); + for (const properties of [refresh, targeted]) { + expect(properties.completeness.enum).toContain('timed_out'); + expect(properties.is_authoritative.type).toBe('boolean'); + expect(properties.provenance).toBeDefined(); + } + expect(refresh.ran.description).toContain('recorded'); + expect(refresh.ran.description).not.toContain('agent_storyboard_status was updated'); + }); +}); diff --git a/server/tests/unit/compliance-publication.test.ts b/server/tests/unit/compliance-publication.test.ts index 9f2ee6cbb30..6096ec26f21 100644 --- a/server/tests/unit/compliance-publication.test.ts +++ b/server/tests/unit/compliance-publication.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { complianceResultToDbInput, type ComplianceResult } from '../../src/addie/services/compliance-testing.js'; +import { classifyComplianceStep } from '../../src/compliance/step-disposition.js'; const agentUrl = 'https://agent.example.test/mcp'; function result(steps: unknown[], completeness: 'complete' | 'timed_out' = 'complete'): ComplianceResult { @@ -22,6 +23,17 @@ describe('hosted compliance publication policy', () => { expect(input.storyboard_statuses?.map(s => s.storyboard_id)).toEqual(['first', 'second', 'third']); }); + it.each(['complete', 'timed_out'] as const)('normalizes and prepares %s runs for persistence with observations omitted', completeness => { + const run = result([{ passed: true }], completeness); + delete (run as Partial).observations; + run.tracks[0].status = 'fail'; + const input = complianceResultToDbInput(run, agentUrl, 'production'); + expect(input).toMatchObject({ completeness, is_authoritative: completeness === 'complete', + overall_status: 'passing', tracks_passed: 1, tracks_failed: 0, observations_json: undefined }); + expect(input.tracks_json[0].status).toBe('pass'); + expect(input.storyboard_statuses?.[0]).toMatchObject({ status: 'passing', steps_total: 1 }); + }); + it('keeps a fixed selected storyboard denominator across rotated execution slices', () => { const first = result([{ passed: true }]); const second = result([{ passed: true }]); @@ -44,4 +56,135 @@ describe('hosted compliance publication policy', () => { .toMatchObject({ is_authoritative: false, replace_storyboard_statuses: false }); }); + it.each([ + 'not_applicable', 'transport_ungradable', 'grader_skipped', + 'capability_profile_mismatch', 'mcp_mode_flattens_url_edges', 'oauth_not_advertised', + 'rate_limit_not_triggered', 'force_scenario_unsupported', 'fixture_seed_unsupported', + 'fixture_unsatisfied', 'capability_unsupported', 'capability_prerequisite_unavailable', + ])('never grades %s as an agent failure, independently of diagnostic text', skip_reason => { + const input = complianceResultToDbInput(result([{ passed: false, skipped: true, skip_reason, error: 'failure' }]), agentUrl, 'production'); + expect(input.storyboard_statuses?.[0]).toMatchObject({ status: 'untested', steps_total: 0 }); + expect(input.tracks_failed).toBe(0); + }); + + it('neutralizes only the synthetic missing-tool gate, keeping production tasks failing', () => { + const step = { passed: false, skipped: true, skip_reason: 'missing_tool' }; + expect(classifyComplianceStep(step, 'first/missing_tool')).toBe('not_applicable'); + expect(classifyComplianceStep({ ...step, step_id: 'missing_tool' }, 'first/missing_tool')).toBe('not_applicable'); + expect(classifyComplianceStep({ ...step, step_id: 'production_task' }, 'first/missing_tool')).toBe('failed'); + expect(classifyComplianceStep(step, 'first/check')).toBe('failed'); + expect(complianceResultToDbInput(result([step]), agentUrl, 'production').storyboard_statuses?.[0].status).toBe('failing'); + }); + + it('uses pinned requires_tool metadata rather than identical missing-tool warning text', () => { + const warning = 'Required tool "sync_governance" not advertised'; + const optional = result([{ passed: true, skipped: true, skip_reason: 'missing_tool', + step_id: 'sync_governance', task: 'sync_governance', warnings: [warning] }]); + optional.bundle_results = undefined; + optional.tracks[0].scenarios[0].scenario = 'media_buy_seller/governance_setup' as never; + const optionalInput = complianceResultToDbInput(optional, agentUrl, 'production'); + expect(optionalInput.storyboard_statuses?.[0].status).toBe('untested'); + expect(optionalInput.tracks_json[0]).toMatchObject({ status: 'skip', has_coverage_gap_skip: false }); + expect(optionalInput.tracks_passed).toBe(0); + expect(optionalInput.overall_status).not.toBe('passing'); + const production = result([{ passed: true, skipped: true, skip_reason: 'missing_tool', + step_id: 'required_production_task', task: 'get_products', warnings: [warning] }]); + expect(complianceResultToDbInput(production, agentUrl, 'production').storyboard_statuses?.[0].status).toBe('failing'); + expect(complianceResultToDbInput(production, agentUrl, 'production').tracks_json[0].has_coverage_gap_skip).toBe(false); + + optional.tracks.push(result([{ passed: true }]).tracks[0]); + const mixedInput = complianceResultToDbInput(optional, agentUrl, 'production'); + expect(mixedInput.overall_status).toBe('passing'); + expect(mixedInput.tracks_passed).toBe(1); + }); + + it('keeps controller prerequisites neutral when the exact pinned step itself requires the absent controller', () => { + const run = result([ + { passed: true, skipped: true, skip_reason: 'missing_test_controller', step_id: 'force_upstream_unavailable', task: 'comply_test_controller' }, + { passed: false, skipped: true, skip_reason: 'prerequisite_failed', step_id: 'stale_response_wire_placement', task: 'get_products', warnings: [] }, + ]); + run.bundle_results = undefined; + run.tracks[0].scenarios[0].scenario = 'stale_response_advisory/stale_response_forcing' as never; + expect(complianceResultToDbInput(run, agentUrl, 'production').storyboard_statuses?.[0]) + .toMatchObject({ status: 'untested', steps_total: 0 }); + expect(complianceResultToDbInput(run, agentUrl, 'production').tracks_json[0]) + .toMatchObject({ status: 'skip', has_coverage_gap_skip: true }); + expect(complianceResultToDbInput(run, agentUrl, 'production').overall_status).not.toBe('passing'); + }); + + it('keeps nested storyboard failures in their selected denominator and isolates sibling fixture gaps', () => { + const run = result([{ skipped: true, skip_reason: 'fixture_unavailable' }]); + run.tracks[0].scenarios[0].scenario = 'media_buy_seller/fixture_setup/check' as never; + run.tracks[0].scenarios.push({ scenario: 'media_buy_seller/available_actions/discover_product_action_template', + overall_passed: false, steps: [ + { step: 'Required production call', passed: false, skipped: true, skip_reason: 'prerequisite_failed' }, + ] } as never); + run.bundle_results![0].storyboard_ids = ['media_buy_seller/fixture_setup', 'media_buy_seller/available_actions']; + const input = complianceResultToDbInput(run, agentUrl, 'production'); + expect(input.storyboard_statuses).toEqual([ + expect.objectContaining({ storyboard_id: 'media_buy_seller/available_actions', status: 'failing', steps_total: 1 }), + expect.objectContaining({ storyboard_id: 'media_buy_seller/fixture_setup', status: 'untested', steps_total: 0 }), + ]); + expect(input.overall_status).not.toBe('passing'); + }); + + it('prefers canonical not_applicable over a future detailed reason', () => { + expect(classifyComplianceStep({ skipped: true, skip_reason: 'new_reason', skip: { reason: 'not_applicable' } })) + .toBe('not_applicable'); + }); + + it('retains genuine dependency failures after a passed producer and unrelated neutral skip', () => { + const input = complianceResultToDbInput(result([ + { passed: true, task: 'get_products' }, + { skipped: true, skip_reason: 'missing_test_controller' }, + { passed: false, skipped: true, skip_reason: 'prerequisite_failed', warnings: [] }, + ]), agentUrl, 'production'); + expect(input.storyboard_statuses?.[0]).toMatchObject({ status: 'partial', steps_total: 2, steps_passed: 1, skipped_count: 1 }); + expect(input.overall_status).not.toBe('passing'); + }); + + it('keeps controller seeding and its cascades as setup gaps, but retains real controller failures', () => { + expect(classifyComplianceStep({ passed: false, task: 'comply_test_controller' }, 'first/__controller_seeding__')).toBe('setup_gap'); + expect(classifyComplianceStep({ skipped: true, skip_reason: 'controller_seeding_failed' })).toBe('setup_gap'); + expect(classifyComplianceStep({ passed: false, task: 'comply_test_controller' }, 'first/check')).toBe('failed'); + expect(classifyComplianceStep({ skipped: true, skip_reason: 'requirement_unmet', requirement: 'request_signer' })).toBe('setup_gap'); + }); + + it('does not promote a skipped dependency track beside a healthy track', () => { + const run = result([{ passed: true }]); + run.tracks.push({ ...run.tracks[0], track: 'signals', status: 'skip', scenarios: [{ + ...run.tracks[0].scenarios[0], scenario: 'second/check' as never, overall_passed: false, + steps: [{ passed: false, skipped: true, skip_reason: 'prerequisite_failed' }], + }] } as never); + expect(complianceResultToDbInput(run, agentUrl, 'production').overall_status).not.toBe('passing'); + }); + + it('retains detached assertion failures when controller setup is neutralized', () => { + // rc.33 projects the seeding title into step and drops step_id. + const run = result([{ step: 'Seed account fixture-account', passed: false, + task: 'comply_test_controller', error: 'Setup unavailable' }]); + run.tracks[0].scenarios[0].scenario = 'first/__controller_seeding__' as never; + run.tracks[0].scenarios[0].overall_passed = false; + run.tracks[0].status = 'fail'; + run.observations = [{ track: 'core', severity: 'error', source: { + kind: 'storyboard', code: 'storyboard-assertion-failed', storyboard_id: 'first', + } }] as never; + const input = complianceResultToDbInput(run, agentUrl, 'production'); + expect(input.overall_status).toBe('failing'); + expect(input.storyboard_statuses?.[0]).toMatchObject({ status: 'failing', failure_count: 1 }); + expect(input.step_diagnostics).toEqual([expect.objectContaining({ + phase_id: '__controller_seeding__', task: 'comply_test_controller', + error_text: 'Setup unavailable', step_passed: false, + })]); + }); + + it('records exact runner and agent provenance, leaving absent agent builds null', () => { + const run = result([{ passed: true }]); + expect(complianceResultToDbInput(run, agentUrl, 'production').provenance_json).toMatchObject({ + compliance_bundle_version: '3.1.20', sdk_version: '14.0.0-rc.33', + agent_build_version: 'test-build', agent_library_version: 'seller-sdk-1', + }); + delete run.agent_profile.adcp_build_version; + expect(complianceResultToDbInput(run, agentUrl, 'production').provenance_json?.agent_build_version).toBeNull(); + }); }); diff --git a/server/tests/unit/compliance-step-diagnostics.test.ts b/server/tests/unit/compliance-step-diagnostics.test.ts index 9bdb3358640..2dd4f4ab8ed 100644 --- a/server/tests/unit/compliance-step-diagnostics.test.ts +++ b/server/tests/unit/compliance-step-diagnostics.test.ts @@ -4,6 +4,7 @@ vi.mock('@adcp/sdk/testing', () => ({ setAgentTesterLogger: vi.fn(), comply: vi.fn(), loadComplianceIndex: vi.fn(() => ({ specialisms: [] })), + getComplianceStoryboardById: vi.fn(() => undefined), SAMPLE_BRIEFS: [], getBriefsByVertical: vi.fn(() => []), })); @@ -22,9 +23,9 @@ import { extractFailingStepDiagnostics, complianceResultToDbInput, } from '../../src/addie/services/compliance-testing.js'; -import { getStoryboard } from '../../src/services/storyboards.js'; +import { getComplianceStoryboardById } from '@adcp/sdk/testing'; -const mockGetStoryboard = vi.mocked(getStoryboard); +const mockGetPinnedStoryboard = vi.mocked(getComplianceStoryboardById); function step(overrides: Record) { return { @@ -53,6 +54,7 @@ function phase(scenarioKey: string, steps: any[]) { function resultWith(tracks: any[]) { return { overall_status: 'partial', + adcp_version: '3.1.20', tracks, summary: { headline: 'fixture', @@ -69,8 +71,34 @@ function resultWith(tracks: any[]) { describe('extractFailingStepDiagnostics', () => { beforeEach(() => { - mockGetStoryboard.mockReset(); - mockGetStoryboard.mockReturnValue(undefined); + mockGetPinnedStoryboard.mockReset(); + mockGetPinnedStoryboard.mockReturnValue(undefined); + }); + + it('loads pinned metadata once for repeated skipped steps across adapter projections', () => { + mockGetPinnedStoryboard.mockReturnValue({ phases: [{ id: 'setup', steps: [{ + id: 'optional', title: 'Optional controller', task: 'comply_test_controller', requires_tool: 'comply_test_controller', + }] }] } as any); + const result = resultWith([{ track: 'core', status: 'skip', duration_ms: 0, + scenarios: [phase('nested/storyboard/setup', Array.from({ length: 5 }, () => ({ + step_id: 'optional', passed: false, skipped: true, skip_reason: 'missing_tool', + })))], + }]) as any; + const input = complianceResultToDbInput(result, 'https://agent.example.test/mcp', 'production'); + expect(input.tracks_json[0].status).toBe('skip'); + expect(input.overall_status).not.toBe('passing'); + expect(mockGetPinnedStoryboard).toHaveBeenCalledTimes(1); + expect(mockGetPinnedStoryboard.mock.calls[0][0]).toBe('nested/storyboard'); + expect(JSON.stringify(mockGetPinnedStoryboard.mock.calls[0][1])).toContain('3.1.20'); + complianceResultToDbInput(result, 'https://agent.example.test/mcp', 'production'); + expect(mockGetPinnedStoryboard).toHaveBeenCalledTimes(1); + }); + + it('leaves unmatched attribution unknown when the pinned metadata is unavailable', () => { + mockGetPinnedStoryboard.mockImplementation(() => { throw new Error('Pinned bundle unavailable'); }); + const result = resultWith([]) as any; + result.failures = [{ track: 'core', storyboard_id: 'unavailable', step_id: 'hidden', task: 'get_products' }]; + expect(extractFailingStepDiagnostics(result)[0].phase_id).toBe('unknown'); }); it('captures request + response payloads for a failing wire step', () => { @@ -385,6 +413,7 @@ describe('extractFailingStepDiagnostics', () => { track: 'media_buy', storyboard_id: 'media_buy_seller/available_actions', step_id: 'get_product_allowed_actions', + adcp_error: { code: 'INVALID_REQUEST', field: 'account' }, step_title: 'Read allowed_actions from get_products', task: 'get_products', error: 'Probe validations failed.', @@ -408,6 +437,7 @@ describe('extractFailingStepDiagnostics', () => { step_id: 'get_product_allowed_actions', task: 'get_products', duration_ms: 456, + adcp_error_jsonb: { code: 'INVALID_REQUEST', field: 'account' }, error_text: 'Probe validations failed.', response_jsonb: { products: [{ product_id: 'available_actions_display', allowed_actions: [] }], @@ -612,7 +642,7 @@ describe('extractFailingStepDiagnostics', () => { }); it('captures an unmatched failure summary when the runner halts after a passing visible step', () => { - mockGetStoryboard.mockReturnValue({ + mockGetPinnedStoryboard.mockReturnValue({ phases: [ { id: 'list_and_filter', @@ -692,7 +722,7 @@ describe('extractFailingStepDiagnostics', () => { }); it('captures an unmatched stale-response failure summary when the visible step has no failed validation', () => { - mockGetStoryboard.mockReturnValue({ + mockGetPinnedStoryboard.mockReturnValue({ phases: [ { id: 'stale_response_forcing', @@ -822,7 +852,7 @@ describe('extractFailingStepDiagnostics', () => { }); it('resolves an unmatched hidden failure to its authored phase instead of the first scenario', () => { - mockGetStoryboard.mockReturnValue({ + mockGetPinnedStoryboard.mockReturnValue({ phases: [ { id: 'discovery', @@ -934,7 +964,7 @@ describe('extractFailingStepDiagnostics', () => { fix_command: 'adcp storyboard step https://x creative_lifecycle phase_two_hidden --json', }, ]; - mockGetStoryboard.mockReturnValue({ + mockGetPinnedStoryboard.mockReturnValue({ phases: [ { id: 'phase_one', diff --git a/server/tests/unit/compliance-testing-effective-run-status.test.ts b/server/tests/unit/compliance-testing-effective-run-status.test.ts index 69e3175f1cb..44a89af9c0e 100644 --- a/server/tests/unit/compliance-testing-effective-run-status.test.ts +++ b/server/tests/unit/compliance-testing-effective-run-status.test.ts @@ -141,7 +141,7 @@ describe('complianceResultToDbInput — effectiveRunStatus', () => { ]); }); - it('does not flag explicit requires_tool skips as coverage gaps', () => { + it('does not infer optional missing tools from warning text without pinned metadata', () => { const result = makeResult([ { track: 'core', @@ -171,9 +171,9 @@ describe('complianceResultToDbInput — effectiveRunStatus', () => { const out = complianceResultToDbInput(result as any, 'https://agent.example.com/mcp', 'production'); - expect(out.overall_status).toBe('passing'); + expect(out.overall_status).toBe('failing'); expect(out.tracks_json).toEqual([ - expect.objectContaining({ track: 'core', status: 'silent', has_coverage_gap_skip: false }), + expect.objectContaining({ track: 'core', status: 'fail', has_coverage_gap_skip: false }), ]); }); @@ -221,7 +221,7 @@ describe('complianceResultToDbInput — effectiveRunStatus', () => { expect.objectContaining({ track: 'core', has_coverage_gap_skip: false }), expect.objectContaining({ track: 'creative', - status: 'pass', + status: 'skip', has_coverage_gap_skip: true, }), ]); diff --git a/server/tests/unit/derive-storyboard-statuses.test.ts b/server/tests/unit/derive-storyboard-statuses.test.ts index 29b546c1c12..63fec1ecafa 100644 --- a/server/tests/unit/derive-storyboard-statuses.test.ts +++ b/server/tests/unit/derive-storyboard-statuses.test.ts @@ -32,6 +32,7 @@ function makeResult( ): ComplianceResult { return { agent_url: 'https://example.test/mcp', + agent_profile: { tools: [] }, overall_status: 'passing', tracks: [ { @@ -109,7 +110,7 @@ describe('deriveStoryboardStatuses', () => { })).toBe(false); }); - it('classifies controller skips and their prerequisite cascades as non-executable', () => { + it('keeps unattributed prerequisite cascades executable despite controller warning text', () => { expect(isNonExecutableCoverageGapScenario({ scenario: 'stale_response_advisory/stale_response_forcing', steps: [ @@ -127,7 +128,7 @@ describe('deriveStoryboardStatuses', () => { warnings: ['Skipped: prior stateful step "force_upstream_unavailable" skipped (missing_test_controller); state never materialized.'], }, ], - })).toBe(true); + })).toBe(false); }); it('does not hide a genuine cascade after an unrelated controller skip', () => { @@ -176,7 +177,7 @@ describe('deriveStoryboardStatuses', () => { ]); }); - it('does not hide a production missing-tool cascade after a neutral missing-tool skip', () => { + it('does not hide a production missing-tool cascade after a neutral capability skip', () => { const result = makeResult([ { scenario: 'mixed_missing_tools/exercise', @@ -185,7 +186,7 @@ describe('deriveStoryboardStatuses', () => { { passed: true, skipped: true, - skip_reason: 'missing_tool', + skip_reason: 'capability_unsupported', step: 'Preview the creative', task: 'preview_creative', warnings: ['Required tool "preview_creative" not advertised; agent tools: [get_products].'], @@ -401,7 +402,7 @@ describe('deriveStoryboardStatuses', () => { }); }); - it('treats an unavailable runner fixture and any legacy cascade as untested', () => { + it('treats the structural whole-storyboard fixture abort and later legacy cascades as untested', () => { const result = makeResult([ { scenario: 'creative_fate_after_cancellation/get_products_brief', @@ -489,6 +490,7 @@ describe('deriveStoryboardStatuses', () => { }, ]); + result.adcp_version = '3.1.20'; const [entry] = deriveStoryboardStatuses(result); expect(entry).toEqual({ @@ -519,6 +521,7 @@ describe('deriveStoryboardStatuses', () => { }, ]); + result.adcp_version = '3.1.20'; const [entry] = deriveStoryboardStatuses(result); expect(entry).toEqual({ @@ -529,7 +532,7 @@ describe('deriveStoryboardStatuses', () => { }); }); - it('excludes prerequisite cascades caused by explicit requires_tool skips', () => { + it('preserves unattributed dependencies after a pinned optional-tool skip', () => { const result = makeResult([ { scenario: 'creative_lifecycle/build_and_preview', @@ -555,13 +558,16 @@ describe('deriveStoryboardStatuses', () => { }, ]); + result.adcp_version = '3.1.20'; const [entry] = deriveStoryboardStatuses(result); expect(entry).toEqual({ storyboard_id: 'creative_lifecycle', - status: 'untested', + status: 'failing', steps_passed: 0, - steps_total: 0, + steps_total: 1, + failure_count: 0, + skipped_count: 1, }); }); diff --git a/static/openapi/registry.yaml b/static/openapi/registry.yaml index d5ed68645be..a8e47a531bc 100644 --- a/static/openapi/registry.yaml +++ b/static/openapi/registry.yaml @@ -739,6 +739,8 @@ components: AgentCompliance: type: object properties: + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" status: type: string enum: @@ -829,6 +831,54 @@ components: - streak_days - last_checked_at - headline + ComplianceRunProvenance: + type: + - object + - "null" + properties: + grading_policy_version: + type: string + compliance_bundle_version: + type: + - string + - "null" + sdk_version: + type: string + agent_build_version: + type: + - string + - "null" + description: Agent-reported adcp.build_version; null when not supplied. Never inferred from the endpoint. + agent_library_version: + type: + - string + - "null" + test_session_id: + type: + - string + - "null" + timeout_ms: + type: + - number + - "null" + storyboard_start_offset: + type: + - number + - "null" + auth_type: + type: + - string + - "null" + required: + - grading_policy_version + - compliance_bundle_version + - sdk_version + - agent_build_version + - agent_library_version + - test_session_id + - timeout_ms + - storyboard_start_offset + - auth_type BadgeRole: type: string enum: @@ -4210,6 +4260,8 @@ components: AgentComplianceDetail: type: object properties: + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" agent_url: type: string requested_compliance_target: @@ -4677,8 +4729,18 @@ components: ComplianceRun: type: object properties: + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" id: type: string + completeness: + type: string + enum: + - complete + - timed_out + - not_completed + is_authoritative: + type: boolean requested_compliance_target: type: - string @@ -12045,6 +12107,31 @@ paths: type: - string - "null" + completeness: + type: + - string + - "null" + enum: + - complete + - timed_out + - not_completed + - null + is_authoritative: + type: boolean + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" + storyboard_statuses: + type: array + items: {} + observations: + type: array + items: {} + diagnostics_visibility: + type: string + enum: + - owner_or_operator + diagnostics_note: + type: string count: type: integer diagnostics: @@ -12054,6 +12141,13 @@ paths: required: - agent_url - run_id + - completeness + - is_authoritative + - provenance + - storyboard_statuses + - observations + - diagnostics_visibility + - diagnostics_note - count - diagnostics "400": @@ -12260,6 +12354,8 @@ paths: is_authoritative: type: boolean description: Only authoritative runs update the public grade and badge state. Incomplete runs preserve the previous complete assessment. + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" run_id: type: string description: Compliance run id written by this refresh. Use with /compliance/diagnostics?run_id=... to inspect failing-step wire evidence. @@ -13796,6 +13892,8 @@ paths: - not_completed is_authoritative: type: boolean + provenance: + $ref: "#/components/schemas/ComplianceRunProvenance" storyboard_status: type: object properties: @@ -13899,6 +13997,7 @@ paths: - run_id - completeness - is_authoritative + - provenance - storyboard_status - diagnostics - total_duration_ms diff --git a/tests/addie/compliance-testing.test.ts b/tests/addie/compliance-testing.test.ts index 553410f6f3b..8e4f0b87a13 100644 --- a/tests/addie/compliance-testing.test.ts +++ b/tests/addie/compliance-testing.test.ts @@ -9,8 +9,8 @@ import { function baseResult(overrides: Partial): ComplianceResult { return { agent_url: 'https://agent.example/mcp', - adcp_version: '3.1.0-rc.12', - agent_profile: {}, + adcp_version: '3.1.20', + agent_profile: { tools: [] }, overall_status: 'partial', tracks: [], tested_tracks: [], @@ -175,7 +175,10 @@ describe('compliance result adapter', () => { const dbInput = complianceResultToDbInput(result, 'https://agent.example/mcp', 'production'); expect(dbInput.tracks_json[0]).toMatchObject({ track: 'core', - has_coverage_gap_skip: false, + // webhook_receiver and the pinned idempotency requires_contract are + // unavailable runner setup. They are neutral but leave coverage gaps. + status: 'skip', + has_coverage_gap_skip: true, }); }); @@ -239,15 +242,20 @@ describe('compliance result adapter', () => { ]); const dbInput = complianceResultToDbInput(result, 'https://agent.example/mcp', 'production'); - expect(dbInput.overall_status).toBe('passing'); + // Canonical not_applicable remains neutral. With no executed steps, it + // cannot manufacture a passing grade or increment the passed-track count. + expect(dbInput.overall_status).toBe('partial'); + expect(dbInput.tracks_passed).toBe(0); + expect(dbInput.tracks_failed).toBe(0); expect(dbInput.tracks_partial).toBe(0); expect(dbInput.tracks_json[0]).toMatchObject({ track: 'core', + status: 'skip', has_coverage_gap_skip: false, }); }); - it('keeps non-idempotency missing runner contracts visible as coverage gaps', () => { + it('keeps unmatched missing runner contracts actionable without pinned setup metadata', () => { const result = baseResult({ tracks: [ { @@ -293,7 +301,8 @@ describe('compliance result adapter', () => { const dbInput = complianceResultToDbInput(result, 'https://agent.example/mcp', 'production'); expect(dbInput.tracks_json[0]).toMatchObject({ track: 'core', - has_coverage_gap_skip: true, + status: 'fail', + has_coverage_gap_skip: false, }); });