diff --git a/.changeset/fix-3385-failed-preparation-evidence.md b/.changeset/fix-3385-failed-preparation-evidence.md new file mode 100644 index 000000000..d0865f83d --- /dev/null +++ b/.changeset/fix-3385-failed-preparation-evidence.md @@ -0,0 +1,9 @@ +--- +'@adcp/sdk': patch +--- + +Preserve producer evidence when prepared fixture discovery fails (#3385). + +Authorized full and standalone reuse now retain fixture-handle resolutions and a `fixture_preparation` diagnostic phase with the producer's errors and actual exchanges. The historical phase is outside graded `phases`, preserving consumer and JUnit failure counts. Protected captures are stripped from its public contexts; denied callers still receive only `PreparedFixturesDeniedError`, while the private audit sink retains failed producer evidence. + +Discovery failures identify the attempted task and configured transport. SDK connection errors are never represented as seller responses, protocol rejections remain protocol failures, and malformed seller pages retain a schema diagnostic against the actual received payload before SDK projection. The terminal failed page is retained even beyond the bounded successful-page evidence history. Captured text payloads redact credential fields. diff --git a/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md b/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md index ac151a917..df1c2fe9b 100644 --- a/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md +++ b/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md @@ -717,6 +717,17 @@ Until a companion `adcontextprotocol/adcp` PR lands all six, no *published* storyboard can legally use any of this. That is a merge dependency, not an implementation gap — and it is why the downgrade story below matters. +When a prepared fixture graph failed discovery, authorized reuse retains its +producer phase as `fixture_preparation` and its handle records as +`fixture_resolutions`. The producer phase is diagnostic history outside graded +`phases`, so it does not add JUnit failures or change consumer grading counts. +Protected capture keys are stripped from its public step contexts; a private +audit sink receives the complete failed producer record after a denial. Failed +discovery responses retain the official client's protocol response when available; +successful-page evidence retains parsed SDK data. HTTP-only A2A evidence includes +its status and an extraction note when the bounded capture was truncated. SDK +connection diagnostics never stand in for a seller response. + ### Output inventory for the amendment Everything a report consumer can newly observe, with where it is emitted and diff --git a/src/lib/protocols/mcp.ts b/src/lib/protocols/mcp.ts index 3fd60da24..b1e81d7ac 100644 --- a/src/lib/protocols/mcp.ts +++ b/src/lib/protocols/mcp.ts @@ -1450,6 +1450,7 @@ export async function callMCPToolWithOAuth(options: MCPCallOptions): Promise { - let text: string; + if (!response.body) return { body: '', bodyTruncated: false }; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ''; + let complete = false; try { - text = await response.text(); + while (true) { + const chunk = await reader.read(); + if (chunk.done) { + text += decoder.decode(); + complete = true; + return { body: text.slice(0, maxBodyBytes), bodyTruncated: text.length > maxBodyBytes }; + } + text += decoder.decode(chunk.value, { stream: true }); + if (text.length > maxBodyBytes) return { body: text.slice(0, maxBodyBytes), bodyTruncated: true }; + } } catch { - return { body: '', bodyTruncated: false }; + return { body: text, bodyTruncated: true }; + } finally { + // A cloned response is a tee. Waiting for cancellation here can deadlock + // until the original body is consumed, which happens only after fetch returns. + if (!complete) void reader.cancel().catch(() => {}); + reader.releaseLock(); } - if (text.length <= maxBodyBytes) return { body: text, bodyTruncated: false }; - return { body: text.slice(0, maxBodyBytes), bodyTruncated: true }; } diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 0afae1134..4d7869bf0 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -53,6 +53,7 @@ import { preparedBindingRegistryFor, PreparedStoryboardFixtures, resolvePreparedFixtures, + isTrustedPreparedFixtures, type PreparedFixtureBinding, type PreparedFixturesDisposition, type PreparedFixturesRefusal, @@ -4816,12 +4817,7 @@ async function executeStoryboardPass( const disposition = prepared.state.disposition; if (disposition.status === 'failed') { if (!callerOwnsClients) await closeScopedConnections(options.protocol); - return buildDiscoveryFailedResult(agentUrls, storyboard, { - step: 'Reuse prepared fixture state', - passed: false, - duration_ms: 0, - error: disposition.detail, - }); + return buildPreparationFailedRunResult(agentUrls, storyboard, prepared.state, disposition.detail); } if (disposition.status === 'unsatisfied') { fixtureUnsatisfied = true; @@ -6956,7 +6952,16 @@ async function emitPreparedFixturesAudit( const sink = options.prepared_fixtures_audit_sink; if (!sink) return; try { - await sink(record); + const state = options.prepared_fixtures; + const auditRecord = + isTrustedPreparedFixtures(state) && state.disposition.status === 'failed' + ? { + ...record, + fixture_resolutions: state.resolutions, + ...(state.phase && { fixture_preparation: state.phase }), + } + : record; + await sink(auditRecord); } catch { // Intentionally swallowed; see above. } @@ -7060,13 +7065,111 @@ function buildPreparedRefusalStepResult( }; } +/** Historical producer evidence does not add graded steps to the consumer run. */ +function preparedPhaseForOutput(phase: StoryboardPhaseResult): StoryboardPhaseResult { + return { ...phase, steps: phase.steps.map(step => ({ ...step, context: withoutProtectedCaptures(step.context) })) }; +} + +/** + * Result for a full run reusing a prepared state that FAILED preparation. + * + * Not `buildDiscoveryFailedResult`: that reports agent capability + * discovery, and reusing a failed preparation is a different failure with + * different evidence. Saying `discovery_failed` with task `''` and no + * validations sent an implementor looking at their transport and auth + * when the actual failure was product discovery, and it dropped the + * producer records the preparation had already collected. + * + * The producer phase is retained separately as diagnostic evidence, with + * protected captures stripped from context. Its errors and exchanges survive, + * and `fixture_resolutions` + * carries the same records `prepared.resolutions` holds. Counts and + * failure meaning are unchanged: this is a failure, never a skip, and + * never `fixture_unsatisfied`. + */ +function buildPreparationFailedRunResult( + agentUrls: string[], + storyboard: Storyboard, + state: PreparedStoryboardFixtures, + detail: string +): StoryboardResult { + const resolutions = state.resolutions; + const producerPhase = state.phase; + // The task that actually failed, read off the producer's own record + // rather than guessed. Absent only when preparation failed before any + // step was recorded. + const failingStep = producerPhase?.steps.find(step => step.passed === false); + const syntheticStep: StoryboardStepResult = { + storyboard_id: storyboard.id, + step_id: 'prepared_fixtures_preparation_failed', + phase_id: 'prepared_fixtures_preparation_failed', + title: 'Storyboard failed: fixture preparation did not resolve', + task: failingStep?.task ?? '', + passed: false, + skipped: false, + duration_ms: 0, + validations: [ + { + check: 'fixture_capture_unbuildable', + passed: false, + description: 'Fixture preparation must resolve before a run can reuse it', + error: detail, + json_pointer: null, + expected: 'a fixture graph that resolved during preparation', + actual: 'preparation_failed', + schema_id: null, + schema_url: null, + // Pre-wire for THIS run: the consumer never dispatched. The + // producer's own exchange is retained in fixture_preparation. + request: null, + response: null, + }, + ], + context: {}, + extraction: { path: 'none' }, + error: detail, + }; + const phases: StoryboardPhaseResult[] = [ + { + phase_id: 'prepared_fixtures_preparation_failed', + phase_title: 'Prepared fixture preparation failed', + passed: false, + steps: [syntheticStep], + duration_ms: 0, + }, + ]; + return { + storyboard_id: storyboard.id, + storyboard_title: storyboard.title, + agent_url: redactOAuthUrlForOutput(agentUrls[0]!), + agent_urls: agentUrls.map(redactOAuthUrlForOutput), + overall_passed: false, + ...(resolutions.length > 0 && { fixture_resolutions: resolutions }), + ...(state.coverageGaps.length > 0 && { coverage_gaps: state.coverageGaps }), + phases, + ...(producerPhase && { fixture_preparation: preparedPhaseForOutput(producerPhase) }), + context: {}, + total_duration_ms: 0, + passed_count: 0, + failed_count: 1, + skipped_count: 0, + runner_capability_version: RUNNER_CAPABILITY_VERSION, + tested_at: new Date().toISOString(), + strict_validation_summary: summarizeStrictValidation(phases), + notices: [], + }; +} + /** Terminal step result carrying a non-resolved preparation disposition. */ function buildPreparedDispositionStepResult( storyboard: Storyboard, found: { step: StoryboardStep; phaseId: string }, context: StoryboardContext, disposition: PreparedFixturesDisposition, - contributions: Iterable + contributions: Iterable, + /** Producer records to republish; a standalone result is the whole run. */ + resolutions: readonly FixtureResolutionRecord[] = [], + producerPhase?: StoryboardPhaseResult ): StoryboardStepResult { const detail = 'detail' in disposition ? disposition.detail : 'fixture preparation did not resolve'; if (disposition.status === 'failed') { @@ -7096,8 +7199,18 @@ function buildPreparedDispositionStepResult( actual: 'preparation_failed', schema_id: null, schema_url: null, + // Pre-wire for THIS step: the consumer never dispatched. The + // producer's own exchange rides on its records below. + request: null, + response: null, }, ], + // The producer's own records, so a standalone consumer sees the same + // fixture-handle evidence a full run carries. It had none before: + // the caller was told preparation failed with no way to see which + // handle, which strategy, or what the seller actually returned. + ...(resolutions.length > 0 && { fixture_resolutions: [...resolutions] }), + ...(producerPhase && { fixture_preparation: preparedPhaseForOutput(producerPhase) }), context: withoutProtectedCaptures(context), extraction: { path: 'none' }, contributions: Array.from(contributions), @@ -7462,7 +7575,15 @@ async function runStoryboardStepBody( // A terminal preparation outcome is carried forward verbatim. A // retry must not be able to launder a failed graph into a freshly // discovered alternative. - return buildPreparedDispositionStepResult(storyboard, found, context, disposition, contributions); + return buildPreparedDispositionStepResult( + storyboard, + found, + context, + disposition, + contributions, + prepared.state.resolutions, + prepared.state.phase + ); } const hydrationViolation = hydratePreparedCaptures(prepared.state, context, protectedWriter, contextProvenance); if (hydrationViolation) { diff --git a/src/lib/testing/storyboard/seeding.ts b/src/lib/testing/storyboard/seeding.ts index 53e65137e..d7262236b 100644 --- a/src/lib/testing/storyboard/seeding.ts +++ b/src/lib/testing/storyboard/seeding.ts @@ -25,8 +25,10 @@ import { resolveAccount, resolveBrand, validateResponseSchema, type TestClient } import { getValidator } from '../../validation/schema-loader'; import { ADCP_VERSION } from '../../version'; import { injectLegacyEnvelopeStatus } from '../../utils/envelope-status-compat'; +import { readExtractionPath, unwrapProtocolResponse } from '../../utils/response-unwrapper'; import { callControllerRaw } from '../test-controller'; import { executeStoryboardTask } from './task-map'; +import { getCapturesFromError, withRawResponseCapture, type RawHttpCapture } from '../../protocols/rawResponseCapture'; import { buildFixtureResolutionSpecs, applyFixtureBindingsToRequest, @@ -37,10 +39,13 @@ import { } from './fixture-resolution'; import { readCollectionWitness, type CollectionSelectionWitness } from './collection-selection'; import { resolvePath } from './path'; -import { redactSecrets } from '../../utils/redact-secrets'; +import { redactSecrets, SECRET_KEY_PATTERN } from '../../utils/redact-secrets'; +import { redactCredentialPatterns } from '../../utils/redact-credential-patterns'; +import { MAX_JSON_DEPTH } from '../../utils/json-depth'; import type { ContextProvenanceEntry, FixtureResolutionAttempt, + RunnerExtractionRecord, RunnerRequestRecord, RunnerResponseRecord, FixtureResolutionRecord, @@ -593,7 +598,24 @@ function discoveryExchangeFor(catalog: DiscoveryCatalog, productId: string | und type DiscoveryCatalogResult = | { ok: true; catalog: DiscoveryCatalog } - | { ok: false; error: string; evidence?: unknown }; + | { + ok: false; + error: string; + evidence?: unknown; + /** + * The actual exchange that failed, so the producer step can record + * truthful wire evidence instead of nothing. `response` is null only + * when the transport threw and no response was ever received — never + * a stand-in for one we did not capture. + */ + exchange?: { + request: RunnerRequestRecord; + response: RunnerResponseRecord | null; + extraction?: RunnerExtractionRecord; + }; + /** AJV detail when the page parsed but failed its response schema. */ + schemaFailure?: { detail: string; schema_id: string; schema_url: string }; + }; /** AdCP 3.2 ordered per-handle seed/discover state machine. */ async function runDeclaredFixtureResolution( @@ -774,6 +796,20 @@ async function runDeclaredFixtureResolution( const attempts: FixtureResolutionAttempt[] = []; let boundIds: FixtureResolutionRecord['seller_ids']; let chosenStrategy: FixtureResolutionRecord['strategy']; + /** + * The strategy whose wire call actually ran, successful or not. + * + * `chosenStrategy` is only assigned when a strategy RESOLVES, so a + * discover-only handle that failed left it undefined and the step + * label fell through to `comply_test_controller` — claiming a + * controller seed for a run whose only traffic was `get_products`. + */ + let attemptedStrategy: FixtureResolutionRecord['strategy']; + /** Truthful wire evidence for a failed discovery, never fabricated. */ + let discoveryFailureEvidence: + | { request: RunnerRequestRecord; response: RunnerResponseRecord | null; extraction?: RunnerExtractionRecord } + | undefined; + let discoverySchemaFailure: Extract['schemaFailure']; let failure: string | undefined; const stepStart = Date.now(); const parentStatus = spec.parentProductHandle ? productStatuses.get(spec.parentProductHandle) : undefined; @@ -802,6 +838,7 @@ async function runDeclaredFixtureResolution( }); continue; } + attemptedStrategy = strategy; try { const controllerRequest = applyFixtureBindingsToRequest( { scenario: call.scenario, params: call.params }, @@ -908,9 +945,12 @@ async function runDeclaredFixtureResolution( }); continue; } + attemptedStrategy = strategy; const discovered = await catalog(); if (!discovered.ok) { failure = discovered.error; + discoveryFailureEvidence = discovered.exchange; + discoverySchemaFailure = discovered.schemaFailure; attempts.push({ strategy, disposition: 'failed', detail: failure, response: discovered.evidence }); break; } @@ -1000,7 +1040,17 @@ async function runDeclaredFixtureResolution( step_id: call.step_id, phase_id: CONTROLLER_SEEDING_PHASE_ID, title: call.title, - task: chosenStrategy === 'discover' ? 'get_products' : 'comply_test_controller', + // The task that ACTUALLY ran, else the one this handle declared. + // Falling back to the controller whenever no strategy RESOLVED + // attributed a failed discover-only handle to a seed call that + // never happened — and a handle abandoned because its parent + // failed makes no call at all, so it is named by the strategy it + // declared rather than by a different tool. Such a step carries + // no request record, so nothing implies a call was made. + task: + (chosenStrategy ?? attemptedStrategy ?? spec.strategies[0]) === 'discover' + ? 'get_products' + : 'comply_test_controller', passed: status !== 'failed', duration_ms: Date.now() - stepStart, // An unbuildable capture gets a `check` discriminator like the other @@ -1027,9 +1077,37 @@ async function runDeclaredFixtureResolution( ...(captureFailures.get(spec.handle)?.exchange ?? {}), }, ] - : [], + : discoverySchemaFailure !== undefined + ? [ + // A page that parsed but failed its response schema is a + // `response_schema` failure, not a capture failure. Using + // the contract's own kind keeps the schema identity where + // consumers already look for it. + { + check: 'response_schema', + passed: false, + description: 'Fixture discovery response must satisfy the get_products response schema', + error: discoverySchemaFailure.detail, + json_pointer: null, + schema_id: discoverySchemaFailure.schema_id, + schema_url: discoverySchemaFailure.schema_url, + expected: discoverySchemaFailure.schema_id, + actual: 'schema_validation_failed', + ...(discoveryFailureEvidence && { + request: discoveryFailureEvidence.request, + response: discoveryFailureEvidence.response, + }), + }, + ] + : [], context, - extraction: { path: 'none' }, + extraction: discoveryFailureEvidence?.extraction ?? { path: 'none' }, + // The REAL exchange the failing strategy made. Absent rather than + // invented when the strategy never reached the wire. + ...(discoveryFailureEvidence && { + request: discoveryFailureEvidence.request, + ...(discoveryFailureEvidence.response && { response_record: discoveryFailureEvidence.response }), + }), ...(failure && { error: failure }), }); } @@ -1346,6 +1424,69 @@ async function executeLegacySeedCall( } } +/** + * `{ schema_id, schema_url }` for the `get_products` response schema. + * + * Mirrors `resolveSchemaIdentity` in `validations.ts`: the id follows the + * `/schemas//.json` convention cached schemas use for + * `$id`, and the url dereferences against the public docs origin so an + * implementor can fetch exactly what the page was graded against. + */ +function resolveDiscoverySchemaIdentity(adcpVersion: string): { schema_id: string; schema_url: string } { + const schemaId = `/schemas/${adcpVersion}/media-buy/get-products-response.json`; + return { schema_id: schemaId, schema_url: `https://adcontextprotocol.org${schemaId}` }; +} + +/** Preserve protocol evidence while also scrubbing credentials inside text payloads. */ +function redactDiscoveryResponse(value: unknown): unknown { + const visit = (node: unknown, depth: number): unknown => { + if (depth > MAX_JSON_DEPTH) return '[redacted]'; + if (typeof node === 'string') { + return (redactCredentialPatterns(node, false) as string).replaceAll('', '[redacted]'); + } + if (Array.isArray(node)) return node.map(child => visit(child, depth + 1)); + if (node && typeof node === 'object') { + return Object.fromEntries(Object.entries(node).map(([key, child]) => [key, visit(child, depth + 1)])); + } + return node; + }; + const pattern = new RegExp( + `${SECRET_KEY_PATTERN.source}|^(private[_-]?key|signing[_-]?key|signature|auth[_-]?token|jwt)$`, + 'i' + ); + return visit(redactSecrets(value, pattern), 0); +} + +/** + * AJV detail for a discovery page that does not satisfy the `get_products` + * response schema, or `undefined` when it does. + */ +function describeDiscoverySchemaFailure( + data: unknown, + options: StoryboardRunOptions +): { error: string; schemaFailure?: { detail: string; schema_id: string; schema_url: string } } | undefined { + try { + const version = options.adcpVersion ?? ADCP_VERSION; + const validator = getValidator('get_products', 'sync', version); + let detail: string; + if (validator) { + const schemaData = isPlainRecord(data) ? injectLegacyEnvelopeStatus(data, { toolName: 'get_products' }) : data; + if (validator(schemaData)) return undefined; + detail = (validator.errors ?? []) + .map(error => `${error.instancePath || '/'} ${error.message ?? error.keyword}`) + .join('; '); + } else { + const schema = validateResponseSchema('get_products', data, options.adcpVersion); + if (schema.passed) return undefined; + detail = schema.error ?? schema.details ?? 'schema validation failed'; + } + return { error: detail, schemaFailure: { detail, ...resolveDiscoverySchemaIdentity(version) } }; + } catch (err) { + // Missing runner schemas are a setup failure, not evidence against the seller. + return { error: `response schema unavailable: ${err instanceof Error ? err.message : String(err)}` }; + } +} + async function discoverProductCatalog( client: TestClient, options: StoryboardRunOptions, @@ -1380,16 +1521,100 @@ async function discoverProductCatalog( ext: { adcp: { creative_wire: 'canonical' } }, }; let result; + let observedResponse: RawHttpCapture | undefined; + const toolResponse = (captures: RawHttpCapture[]) => + [...captures].reverse().find(capture => capture.requestAdcpSkill === 'get_products'); const pageStart = Date.now(); + // Recorded BEFORE dispatch. A transport failure never reaches the + // response, but the request it sent is real and is the only evidence + // an implementor has to reproduce the failure. + const requestRecord: RunnerRequestRecord = { + transport: options.protocol === 'a2a' ? 'a2a' : 'mcp', + operation: 'get_products', + payload: redactSecrets(request), + }; try { - result = await executeStoryboardTask(client, 'get_products', request, { signal: options.signal }); + // MCP's official SDK trace already supplies its parsed protocol response. + // Capturing its HTTP streams would buffer an unrelated SSE subscription. + if (options.protocol === 'a2a') { + const captured = await withRawResponseCapture( + () => executeStoryboardTask(client, 'get_products', request, { signal: options.signal }), + { maxBodyBytes: 65_536, maxCaptures: 8, maxTotalBodyBytes: 262_144 } + ); + result = captured.result; + observedResponse = toolResponse(captured.captures); + } else { + result = await executeStoryboardTask(client, 'get_products', request, { signal: options.signal }); + } } catch (err) { + const observed = toolResponse(getCapturesFromError(err) ?? []); return { ok: false, - error: `fixture discovery transport failure: ${err instanceof Error ? err.message : String(err)}`, + error: `fixture discovery transport failure: ${redactCredentialPatterns(err instanceof Error ? err.message : String(err), false)}`, + evidence: { page_count: pageCount + 1, request: requestRecord.payload }, + // Explicitly null: no response was received. Not a placeholder. + exchange: { + request: requestRecord, + response: observed + ? { + transport: requestRecord.transport, + payload: redactDiscoveryResponse(observed.body), + status: observed.status, + duration_ms: Date.now() - pageStart, + } + : null, + }, }; } pageCount++; + // The official clients retain the received protocol response before SDK + // validation/projection. A connection error has no such response entry. + let wireResponse = result.debug_logs + ?.slice() + .reverse() + .find( + log => + typeof log.message === 'string' && + (log.message.startsWith('MCP: Tool get_products response received') || + log.message.startsWith('A2A: Response received')) && + log.response !== undefined + )?.response; + if (wireResponse === undefined && observedResponse && !observedResponse.bodyTruncated) { + try { + wireResponse = JSON.parse(observedResponse.body); + } catch { + // Raw diagnostics remain available; protocol decoding belongs to the SDK. + } + } + // An executor error is SDK diagnostic text, not a seller response. Retain + // a response only when a tool body or actual HTTP exchange was observed. + if ( + (!result.success || result.data === undefined) && + result._extraction_path === undefined && + observedResponse === undefined && + wireResponse === undefined + ) { + return { + ok: false, + error: `fixture discovery get_products rejected: ${redactCredentialPatterns(result.error ?? result.adcp_error?.code ?? 'unknown error', false)}`, + evidence: { page_count: pageCount, request: requestRecord.payload }, + exchange: { request: requestRecord, response: null }, + }; + } + const extraction: RunnerExtractionRecord = { + path: result._extraction_path ?? 'none', + ...(observedResponse?.bodyTruncated && { note: 'Raw HTTP response capture was truncated at its body budget.' }), + }; + const failedExchange = { + request: requestRecord, + extraction, + response: { + transport: requestRecord.transport, + payload: redactDiscoveryResponse(wireResponse ?? observedResponse?.body ?? result.data), + ...(observedResponse && { status: observedResponse.status }), + duration_ms: Date.now() - pageStart, + }, + }; // Recorded for EVERY page, success or not: a schema-invalid or rejected // page is exactly the exchange a reader needs to see. // The request is tiny and always retained. The RESPONSE is a second @@ -1401,71 +1626,86 @@ async function discoverProductCatalog( const retainResponse = exchanges.length < DISCOVERY_RETAINED_RESPONSE_PAGES; const pageIndex = exchanges.push({ - request: { transport: 'mcp', operation: 'get_products', payload: redactSecrets(request) }, - response: { - transport: 'mcp', - payload: retainResponse - ? redactSecrets( - result.data ?? - (result.adcp_error ? { adcp_error: result.adcp_error } : undefined) ?? - result.error ?? - null - ) - : { + request: requestRecord, + response: retainResponse + ? { ...failedExchange.response, payload: redactSecrets(result.data) } + : { + transport: requestRecord.transport, + payload: { response_not_retained: `page ${exchanges.length + 1} is past the runner's ${DISCOVERY_RETAINED_RESPONSE_PAGES}-page discovery-evidence budget; the request for this page is recorded in full`, }, - duration_ms: Date.now() - pageStart, - }, + duration_ms: Date.now() - pageStart, + }, }) - 1; + // Graded whether or not the executor accepted the page. A seller that + // returns a malformed body makes the executor reject in strict mode, + // and classifying that as a bare protocol error hid WHICH of the two + // it was — the one thing a `response_schema` record exists to say. + let schemaData = result.success ? result.data : undefined; + if (!result.success && wireResponse !== undefined && result._extraction_path !== 'error') { + try { + schemaData = unwrapProtocolResponse(wireResponse, undefined, options.protocol === 'a2a' ? 'a2a' : 'mcp'); + extraction.path = readExtractionPath(schemaData) ?? extraction.path; + } catch { + // Protocol decoding errors retain their original classification. + schemaData = undefined; + } + } + const nativeResult = + isPlainRecord(wireResponse) && isPlainRecord(wireResponse.result) ? wireResponse.result : undefined; + const nativeState = nativeResult && isPlainRecord(nativeResult.status) ? nativeResult.status.state : undefined; + const sellerRejected = + result._extraction_path === 'error' || + (isPlainRecord(schemaData) && + (schemaData.adcp_error !== undefined || schemaData.status === 'failed' || schemaData.status === 'rejected')) || + nativeState === 'failed' || + nativeState === 'rejected' || + nativeState === 'canceled'; + const schemaCheck = + sellerRejected || schemaData === undefined ? undefined : describeDiscoverySchemaFailure(schemaData, options); if (!result.success) { return { ok: false, - error: `fixture discovery get_products rejected: ${result.error ?? result.adcp_error?.code ?? 'unknown error'}`, - evidence: { page_count: pageCount, request }, + error: `fixture discovery get_products rejected: ${redactCredentialPatterns(result.error ?? result.adcp_error?.code ?? 'unknown error', false)}`, + evidence: { page_count: pageCount, request: requestRecord.payload }, + exchange: failedExchange, + ...(schemaCheck?.schemaFailure && { schemaFailure: schemaCheck.schemaFailure }), }; } - let schemaFailure: string | undefined; - try { - const validator = getValidator('get_products', 'sync', options.adcpVersion ?? ADCP_VERSION); - if (validator) { - const responseData: unknown = result.data; - const schemaData = isPlainRecord(responseData) - ? injectLegacyEnvelopeStatus(responseData, { toolName: 'get_products' }) - : responseData; - if (!validator(schemaData)) { - schemaFailure = (validator.errors ?? []) - .map(error => `${error.instancePath || '/'} ${error.message ?? error.keyword}`) - .join('; '); - } - } else { - const schema = validateResponseSchema('get_products', result.data, options.adcpVersion); - if (!schema.passed) schemaFailure = schema.error ?? schema.details ?? 'schema validation failed'; - } - } catch (err) { - schemaFailure = `response schema unavailable: ${err instanceof Error ? err.message : String(err)}`; - } - if (schemaFailure) { + if (schemaCheck) { return { ok: false, - error: `fixture discovery returned malformed get_products response: ${schemaFailure}`, + error: `fixture discovery get_products validation failed: ${schemaCheck.error}`, evidence: { page_count: pageCount }, + exchange: failedExchange, + ...(schemaCheck.schemaFailure && { schemaFailure: schemaCheck.schemaFailure }), }; } if (!isPlainRecord(result.data)) { - return { ok: false, error: 'fixture discovery returned a non-object get_products response' }; + return { + ...{ ok: false, error: 'fixture discovery returned a non-object get_products response' }, + exchange: failedExchange, + }; } const pageProducts = result.data.products; if (pageProducts !== undefined && !Array.isArray(pageProducts)) { - return { ok: false, error: 'fixture discovery get_products.products is not an array' }; + return { + ...{ ok: false, error: 'fixture discovery get_products.products is not an array' }, + exchange: failedExchange, + }; } for (const product of pageProducts ?? []) { if (!isPlainRecord(product) || typeof product.product_id !== 'string' || product.product_id.length === 0) { - return { ok: false, error: 'fixture discovery returned a product without a non-empty product_id' }; + return { + ...{ ok: false, error: 'fixture discovery returned a product without a non-empty product_id' }, + exchange: failedExchange, + }; } if (seenProductIds.has(product.product_id)) { return { ok: false, error: `fixture discovery returned duplicate product_id "${product.product_id}"; response order cannot resolve duplicate identities`, + exchange: failedExchange, }; } seenProductIds.add(product.product_id); @@ -1477,6 +1717,7 @@ async function discoverProductCatalog( return { ok: false, error: `fixture discovery returned duplicate pricing option identity "${product.product_id}\u0000${option.pricing_option_id}"`, + exchange: failedExchange, }; } seenPricingIds.add(option.pricing_option_id); @@ -1485,17 +1726,26 @@ async function discoverProductCatalog( pageOfProduct.set(product.product_id, pageIndex); products.push(product); if (products.length > 100_000) { - return { ok: false, error: 'fixture discovery exceeded the 100,000-product safety limit' }; + return { + ...{ ok: false, error: 'fixture discovery exceeded the 100,000-product safety limit' }, + exchange: failedExchange, + }; } } const pagination = isPlainRecord(result.data.pagination) ? result.data.pagination : undefined; if (pagination?.has_more === true) { const next = pagination.cursor; if (typeof next !== 'string' || next.length === 0) { - return { ok: false, error: 'fixture discovery pagination is broken: has_more=true without cursor' }; + return { + ...{ ok: false, error: 'fixture discovery pagination is broken: has_more=true without cursor' }, + exchange: failedExchange, + }; } if (seenCursors.has(next)) { - return { ok: false, error: `fixture discovery pagination is broken: repeated cursor "${next}"` }; + return { + ...{ ok: false, error: `fixture discovery pagination is broken: repeated cursor "${next}"` }, + exchange: failedExchange, + }; } seenCursors.add(next); cursors.push(next); diff --git a/src/lib/testing/storyboard/task-map.ts b/src/lib/testing/storyboard/task-map.ts index 1eb411457..2039551e4 100644 --- a/src/lib/testing/storyboard/task-map.ts +++ b/src/lib/testing/storyboard/task-map.ts @@ -8,6 +8,7 @@ */ import type { TaskResult } from '../types'; +import type { DebugLogEntry } from '../../types/adcp'; import type { AdcpErrorInfo } from '../../core/ConversationTypes'; import { isTerminalAdcpError, readExtractionPath } from '../../utils/response-unwrapper'; import { generateIdempotencyKey } from '../../utils/idempotency'; @@ -316,7 +317,7 @@ export async function executeStoryboardTask( taskName: string, params: Record, opts: StoryboardTaskExecutionOptions = {} -): Promise { +): Promise { const unresolvedAssetDirectives = findUnresolvedCreativeAssetDirectives(params); if (unresolvedAssetDirectives.length > 0) { return { diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index 17eca9a38..5059f4d00 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -2953,6 +2953,18 @@ export interface StoryboardStepResult { agent_url?: string; /** 1-based index of the agent instance (multi-instance mode). Absent in single-URL mode. */ agent_index?: number; + /** + * Producer fixture-handle records, when this step result stands in for a + * whole run. + * + * `runStoryboardStep` returns one step as the entire result, so a + * standalone caller reusing a prepared state has nowhere else to read + * the `fixture_resolutions` block a full run carries. Present only when + * the prepared state actually holds records. + */ + fixture_resolutions?: FixtureResolutionRecord[]; + /** Historical producer evidence; excluded from consumer grading and JUnit counts. */ + fixture_preparation?: StoryboardPhaseResult; /** Exact request the runner sent (contract-required on failures). */ request?: RunnerRequestRecord; /** Exact response observed, including transport, status, headers. */ @@ -3446,6 +3458,8 @@ export interface StoryboardResult { overall_passed: boolean; /** AdCP 3.2 fixture-handle setup/discovery evidence, one row per handle. */ fixture_resolutions?: FixtureResolutionRecord[]; + /** Historical producer evidence; excluded from consumer grading and JUnit counts. */ + fixture_preparation?: StoryboardPhaseResult; /** Storyboard-level fixture coverage gaps, emitted once per cause. */ coverage_gaps?: FixtureResolutionCoverageGap[]; /** diff --git a/test/lib/mcp-oauth-scoped-fetch.test.js b/test/lib/mcp-oauth-scoped-fetch.test.js index f9ddcba8c..5c9ddeddf 100644 --- a/test/lib/mcp-oauth-scoped-fetch.test.js +++ b/test/lib/mcp-oauth-scoped-fetch.test.js @@ -576,14 +576,17 @@ test('modern OAuth refresh uses the scoped fetcher instead of global fetch', asy const server = await startOAuthServer('modern'); try { await withGlobalFetchGuard(async (fetchFn, fetchedUrls) => { + const debugLogs = []; const result = await callMCPToolWithOAuth({ agentUrl: server.url, toolName: 'ping', args: {}, authProvider: createRefreshProvider(server.state.origin), fetchFn, + debugLogs, }); assert.equal(result.content[0].text, 'pong'); + assert.deepEqual(debugLogs.find(log => log.message === 'MCP: Tool ping response received').response, result); assert.ok( fetchedUrls.some(url => url.endsWith('/token')), 'scoped fetcher should perform the token exchange' diff --git a/test/lib/raw-response-capture.test.js b/test/lib/raw-response-capture.test.js index 84c1da254..8aaf30169 100644 --- a/test/lib/raw-response-capture.test.js +++ b/test/lib/raw-response-capture.test.js @@ -149,6 +149,28 @@ describe('rawResponseCapture', () => { assert.doesNotMatch(JSON.stringify(captures[0]), /must-not-be-captured/); }); + test('bounded capture returns without waiting for an oversized stream to finish', { timeout: 2000 }, async t => { + let controller; + const stream = new ReadableStream({ + start(value) { + controller = value; + value.enqueue(new TextEncoder().encode('x'.repeat(1024))); + }, + }); + t.after(() => { + try { + controller.close(); + } catch {} + }); + const capturingFetch = wrapFetchWithCapture(async () => new Response(stream)); + const { result, captures } = await withRawResponseCapture(() => capturingFetch('https://example.test/'), { + maxBodyBytes: 32, + }); + assert.equal(captures[0].body.length, 32); + assert.equal(captures[0].bodyTruncated, true); + await result.body.cancel(); + }); + test('truncates body when it exceeds maxBodyBytes', async () => { const big = 'A'.repeat(10_000); const { server, url } = await startServer((req, res) => { diff --git a/test/lib/storyboard-prepared-fixtures.test.js b/test/lib/storyboard-prepared-fixtures.test.js index 25d99ca8d..6074c78f1 100644 --- a/test/lib/storyboard-prepared-fixtures.test.js +++ b/test/lib/storyboard-prepared-fixtures.test.js @@ -25,6 +25,7 @@ const { prepareStoryboardFixtures, runStoryboard, runStoryboardStep, + withPreparedFixtures, } = require('../../dist/lib/testing/index.js'); const CAPS = { @@ -69,6 +70,35 @@ function product(id, extra = {}) { } function transport(catalog, overrides = {}) { + // `overrides.transportFailure` makes the fetch itself reject, so a test + // can exercise the transport arm without a response ever existing. + if (overrides.transportFailure) { + const wire = []; + const calls = []; + return { + calls, + wire, + options: { + adcpVersion: '3.2.5', + timeout_ms: 5000, + profile: { tools: TOOLS, raw_capabilities: CAPS, supported_protocols: ['media_buy'] }, + transport: { + trustedFetchFn: async (_input, init = {}) => { + const message = JSON.parse(init.body); + wire.push(message.method); + if (message.method === 'tools/call' && message.params?.name === 'get_products') { + throw new Error('synthetic transport failure: connection reset'); + } + return originalTransport(catalog).options.transport.trustedFetchFn(_input, init); + }, + }, + }, + }; + } + return originalTransport(catalog, overrides); +} + +function originalTransport(catalog, overrides = {}) { const calls = []; // Every seller endpoint interaction, including official-client // initialization and tool discovery — not just the mutating calls. A gate @@ -98,6 +128,33 @@ function transport(catalog, overrides = {}) { } else if (message.method === 'tools/call') { const { name, arguments: request } = message.params; calls.push({ task: name, request }); + if (name === 'get_products' && overrides.protocolFailure) { + return new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { + isError: true, + content: [ + { + type: 'text', + text: JSON.stringify( + overrides.protocolPayload ?? { + status: 'failed', + adcp_error: { + code: 'INVALID_REQUEST', + message: 'Seller refused discovery', + recovery: 'terminal', + }, + } + ), + }, + ], + }, + }), + { headers: { 'content-type': 'application/json' } } + ); + } let data; if (name === 'get_adcp_capabilities') data = CAPS; else if (name === 'get_products') @@ -107,6 +164,7 @@ function transport(catalog, overrides = {}) { cache_scope: 'account', wholesale_feed_version: 'v1', }; + else if (name === 'comply_test_controller' && overrides.controller) data = overrides.controller(request); else if (name === 'create_media_buy') data = { status: 'completed', @@ -2036,6 +2094,415 @@ describe('refused caller gets a denial; the host gets the evidence', () => { }); }); +describe('failed preparation keeps its producer evidence (#3385)', () => { + /** + * The four ways discovery can fail, each with the shape that produces + * it. Transport never reaches a response; the other three do. + */ + const FAILURE_MODES = { + transport: { transportFailure: true }, + protocol: { protocolFailure: true }, + schema: { read: () => ({ status: 'completed', products: [{ product_id: 'malformed' }] }) }, + pagination: { + read: () => ({ + status: 'completed', + products: [SELECTED], + cache_scope: 'account', + wholesale_feed_version: 'v1', + // has_more with no cursor: the seller promised another page and + // gave the runner no way to ask for it. + pagination: { has_more: true }, + }), + }, + }; + + /** Prepare against a seller that fails discovery in the named way. */ + async function prepareFailing(mode) { + const io = transport([SELECTED], FAILURE_MODES[mode]); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed', `${mode}: ${JSON.stringify(state.disposition)}`); + assert.ok(state.resolutions.length > 0, `${mode}: the producer recorded its handles`); + return { state, io }; + } + + for (const mode of Object.keys(FAILURE_MODES)) { + test(`${mode}: a full run reuse reports product discovery and keeps the producer records`, async () => { + const { state } = await prepareFailing(mode); + const io = transport([SELECTED]); + const result = await runStoryboard(ENDPOINT, board(), { + ...io.options, + ...withPreparedFixtures(state, ALLOW), + }); + assert.equal(result.overall_passed, false); + assert.equal(result.failed_count, 1); + assert.equal(result.skipped_count, 0, 'a failed preparation is a failure, never a skip'); + + // The producer records survive into the consumer's result. + assert.deepEqual(result.fixture_resolutions, state.resolutions); + + // The failure is named as product discovery, not capability discovery. + const steps = result.phases.flatMap(phase => phase.steps); + const synthetic = steps.find(step => step.step_id === 'prepared_fixtures_preparation_failed'); + assert.ok(synthetic, JSON.stringify(result.phases.map(p => p.phase_id))); + assert.equal(synthetic.task, 'get_products', 'the real failing task, not a blank capability row'); + assert.equal( + steps.some(step => step.step_id === 'discovery_failed'), + false, + 'a failed preparation is not an agent capability-discovery failure' + ); + const check = synthetic.validations.find(v => v.check === 'fixture_capture_unbuildable'); + assert.ok(check, JSON.stringify(synthetic.validations)); + assert.equal(check.actual, 'preparation_failed'); + + // The producer's own phase, with its original context and error. + const producer = result.fixture_preparation; + assert.ok(producer, 'the producer phase is retained separately from consumer grading'); + assert.ok( + producer.steps.some(step => step.passed === false), + 'with its failing step intact' + ); + assert.equal( + result.phases.flatMap(phase => phase.steps).filter(step => !step.passed && !step.skipped).length, + result.failed_count + ); + const { formatStoryboardResultsAsJUnit } = require('../../dist/lib/testing/storyboard/junit.js'); + assert.match(formatStoryboardResultsAsJUnit([result]), /failures="1"/); + assert.deepEqual( + producer.steps.map(step => step.error), + state.phase.steps.map(step => step.error) + ); + }); + + test(`${mode}: a standalone reuse carries the same producer records`, async () => { + const { state } = await prepareFailing(mode); + const io = transport([SELECTED]); + const result = await runStoryboardStep(ENDPOINT, board(), 'create', { + ...io.options, + ...withPreparedFixtures(state, ALLOW), + }); + assert.equal(result.passed, false); + assert.deepEqual(result.fixture_resolutions, state.resolutions); + assert.ok(result.fixture_preparation); + const check = result.validations.find(v => v.check === 'fixture_capture_unbuildable'); + assert.ok(check, JSON.stringify(result.validations)); + assert.equal(check.actual, 'preparation_failed'); + assert.equal(check.request, null); + assert.equal(check.response, null); + // No consumer exchange occurred, so none is invented. + assert.equal( + io.calls.some(call => call.task === 'create_media_buy'), + false + ); + }); + + test(`${mode}: the producer step names get_products and never claims a controller seed`, async () => { + const { state } = await prepareFailing(mode); + const producerSteps = state.phase.steps; + const failing = producerSteps.find(step => step.passed === false); + assert.ok(failing, JSON.stringify(producerSteps.map(s => ({ id: s.step_id, passed: s.passed })))); + assert.equal(failing.task, 'get_products', 'the wire call was get_products'); + assert.equal( + producerSteps.some(step => step.task === 'comply_test_controller'), + false, + 'no controller seed happened, so none may be claimed' + ); + }); + } + + test('a failed discovery records the real request, and a response only when one arrived', async () => { + for (const mode of Object.keys(FAILURE_MODES)) { + const { state } = await prepareFailing(mode); + const failing = state.phase.steps.find(step => step.passed === false); + assert.ok(failing.request, `${mode}: the dispatched request is recorded`); + assert.equal(failing.request.operation, 'get_products', mode); + assert.equal(failing.request.payload.buying_mode, 'wholesale', mode); + if (mode === 'transport') { + assert.equal(failing.response_record, undefined, 'an SDK connection error is not a seller response'); + continue; + } + assert.ok(failing.response_record, `${mode}: an actual response is recorded`); + assert.equal(failing.response_record.transport, 'mcp', mode); + } + }); + + test('a schema failure carries a response_schema check with the schema identity', async () => { + const { state } = await prepareFailing('schema'); + const failing = state.phase.steps.find(step => step.passed === false); + const check = failing.validations.find(v => v.check === 'response_schema'); + assert.ok(check, JSON.stringify(failing.validations)); + assert.equal(check.passed, false); + assert.equal(check.actual, 'schema_validation_failed'); + assert.equal(check.schema_id, '/schemas/3.2.5/media-buy/get-products-response.json'); + assert.equal(check.schema_url, 'https://adcontextprotocol.org/schemas/3.2.5/media-buy/get-products-response.json'); + assert.ok(check.request, 'the schema failure cites the request it graded'); + assert.ok(check.response, 'and the response it graded'); + }); + + test('a real executor throw records a request without inventing a response', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => { + throw new Error('executor failed before any response'); + }; + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { ...transport([SELECTED]).options, agentTools: ['get_products'] }, + {} + ); + const failing = result.phase.steps.find(step => !step.passed); + assert.equal(failing.request.operation, 'get_products'); + assert.equal(failing.response_record, undefined); + }); + + test('A2A discovery evidence records the configured transport', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => ({ + success: true, + status: 'completed', + data: { + status: 'completed', + products: [{ product_id: 'malformed' }], + }, + }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { ...transport([SELECTED]).options, protocol: 'a2a', agentTools: ['get_products'] }, + {} + ); + const failing = result.phase.steps.find(step => !step.passed); + assert.equal(failing.request.transport, 'a2a'); + assert.equal(failing.response_record.transport, 'a2a'); + }); + + test('protocol and pagination failures are not relabelled as schema failures', async () => { + for (const mode of ['protocol', 'pagination']) { + const { state } = await prepareFailing(mode); + assert.equal( + state.phase.steps.flatMap(step => step.validations).some(v => v.check === 'response_schema'), + false, + mode + ); + } + }); + + test('the terminal failing page is retained even beyond the successful-page evidence budget', async () => { + let pages = 0; + const io = transport([], { + read: () => { + pages++; + return pages <= 16 + ? { + status: 'completed', + products: [], + cache_scope: 'account', + wholesale_feed_version: 'v1', + pagination: { has_more: true, cursor: `page-${pages + 1}` }, + } + : { status: 'completed', products: [{ product_id: 'late-malformed' }] }; + }, + }); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed'); + assert.equal(pages, 17); + const failing = state.phase.steps.find(step => !step.passed); + assert.equal(failing.request.payload.pagination.cursor, 'page-17'); + assert.match(JSON.stringify(failing.response_record.payload), /late-malformed/); + assert.doesNotMatch(JSON.stringify(failing.response_record.payload), /response_not_retained/); + }); + + test('captured protocol text redacts secret fields before producer evidence is retained', async () => { + const io = transport([], { + protocolFailure: true, + protocolPayload: { + status: 'failed', + adcp_error: { + code: 'INVALID_REQUEST', + message: 'Denied', + recovery: 'terminal', + details: { token: 'wire-secret-value', private_key: 'private-key-value', signature: 'signature-value' }, + }, + }, + }); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed'); + assert.doesNotMatch(JSON.stringify(state.phase), /wire-secret-value|private-key-value|signature-value/); + assert.match(JSON.stringify(state.phase), /redacted/); + }); + + test('A2A failed tasks retain the seller rejection without grading a product schema', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const response = { + jsonrpc: '2.0', + id: 'a2a-error', + result: { + kind: 'task', + id: 'failed-discovery', + contextId: 'ctx', + status: { state: 'failed' }, + artifacts: [ + { + artifactId: 'error', + parts: [ + { + kind: 'data', + data: { + adcp_error: { code: 'INVALID_REQUEST', message: 'Seller refused discovery', recovery: 'terminal' }, + }, + }, + ], + }, + ], + }, + }; + const dispatch = async () => ({ + success: false, + error: 'Seller refused discovery', + data: {}, + debug_logs: [{ message: 'A2A: Response received', response }], + }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { + ...transport([]).options, + protocol: 'a2a', + agentTools: ['get_products'], + }, + {} + ); + const failed = result.phase.steps.find(step => !step.passed); + assert.equal(failed.task, 'get_products'); + assert.deepEqual(failed.response_record.payload, response); + assert.equal( + failed.validations.some(v => v.check === 'response_schema'), + false + ); + }); + + test('schema diagnostics use the discovery route version and the terminal SDK trace', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const malformed = { status: 'completed', products: [{ product_id: 'terminal-malformed' }] }; + const protocolResponse = { content: [{ type: 'text', text: JSON.stringify(malformed) }] }; + const dispatch = async () => ({ + success: false, + error: 'validation failed', + data: { products: [] }, + debug_logs: [ + { + message: 'MCP: Tool get_products response received', + response: { content: [{ type: 'text', text: '{"status":"submitted"}' }] }, + }, + { message: 'MCP: Tool get_products response received', response: protocolResponse }, + ], + }); + const client = { getProducts: dispatch, getProductsLegacy: dispatch }; + const options = { ...transport([]).options, adcpVersion: '3.2.6', agentTools: ['get_products'] }; + const result = await runControllerSeeding(client, board(), options, {}, client, () => ({ + client, + options: { ...options, adcpVersion: '3.2.5' }, + })); + const failed = result.phase.steps.find(step => !step.passed); + const check = failed.validations.find(v => v.check === 'response_schema'); + assert.equal(check.schema_id, '/schemas/3.2.5/media-buy/get-products-response.json'); + assert.equal(check.expected, check.schema_id); + assert.deepEqual(failed.response_record.payload, protocolResponse); + }); + + test('SDK-projected errors without wire evidence are never graded as seller schema failures', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => ({ success: false, error: 'connection failed', data: { products: [] } }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { + ...transport([]).options, + agentTools: ['get_products'], + }, + {} + ); + const failed = result.phase.steps.find(step => !step.passed); + assert.equal(failed.response_record, undefined); + assert.equal( + failed.validations.some(v => v.check === 'response_schema'), + false + ); + }); + + test('a partial graph keeps protected captures out of consumer context without duplicating failures', async () => { + const doc = board(); + doc.fixtures.products.push(product('other-product')); + doc.fixture_resolution.products.push({ handle: 'other-product', strategies: ['seed'] }); + const io = transport([SELECTED], { controller: () => ({ success: false, error: 'UPSTREAM_FAILURE' }) }); + const state = await prepareStoryboardFixtures(ENDPOINT, doc, { + ...io.options, + agentTools: [...TOOLS, 'comply_test_controller'], + }); + assert.equal(state.disposition.status, 'failed', JSON.stringify(state.disposition)); + assert.ok( + state.phase.steps.some(step => step.passed), + 'the graph must contain earlier successful producer work' + ); + const runIo = transport([SELECTED]); + const result = await runStoryboard(ENDPOINT, doc, { ...runIo.options, ...withPreparedFixtures(state, ALLOW) }); + assert.equal(result.failed_count, 1); + assert.equal(result.phases.flatMap(phase => phase.steps).length, 1); + assert.doesNotMatch(JSON.stringify(result.fixture_preparation.steps.map(step => step.context)), /__adcp_fixture_/); + assert.deepEqual( + result.fixture_preparation.steps.map(step => step.error), + state.phase.steps.map(step => step.error) + ); + }); + + test('standalone denial retains failed producer evidence only in the private sink', async () => { + const { state } = await prepareFailing('schema'); + const io = transport([SELECTED]); + let audited; + await assert.rejects( + runStoryboardStep(ENDPOINT, board(), 'create', { + ...io.options, + prepared_fixtures: state, + prepared_fixtures_authorizer: () => ({ authorized: false, detail: 'denied' }), + prepared_fixtures_audit_sink: record => { + audited = record; + }, + }), + error => { + assert.ok(error instanceof PreparedFixturesDeniedError); + assert.doesNotMatch(JSON.stringify(error), /malformed|authored-product/); + return error.refusalCode === 'authorizer_denied'; + } + ); + assert.deepEqual(audited.fixture_resolutions, state.resolutions); + assert.deepEqual(audited.fixture_preparation, state.phase); + assert.deepEqual(io.wire, []); + }); + + test('a denied caller still gets only the typed denial, with the record to the sink', async () => { + const { state } = await prepareFailing('schema'); + const io = transport([SELECTED]); + let audited; + await assert.rejects( + () => + runStoryboard(ENDPOINT, board(), { + ...io.options, + prepared_fixtures: state, + prepared_fixtures_audit_sink: record => { + audited = record; + }, + prepared_fixtures_authorizer: () => ({ authorized: false, detail: 'denied' }), + }), + error => error instanceof PreparedFixturesDeniedError && error.refusalCode === 'authorizer_denied' + ); + // A failed preparation does not downgrade the denial: the caller gets + // no producer records, and the host's copy is complete. + assert.ok(audited, 'the sink received the record'); + assert.deepEqual(audited.fixture_resolutions, state.resolutions); + assert.deepEqual(audited.fixture_preparation, state.phase); + assert.deepEqual(io.wire, []); + }); +}); + describe('a capture the funnel refuses to detach', () => { /** A storyboard whose product fixture snapshots the whole pinned entity. */ function snapshotBoard() {