From b3fe076c7db11f7db66ca0af012322ba26dc21d5 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 23 Oct 2026 02:26:48 +0000 Subject: [PATCH 1/2] fix(testing): keep producer evidence when fixture preparation fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #3385. A `prepareStoryboardFixtures()` that ended in `disposition.failed` lost its evidence the moment an authorized `runStoryboard` / `runStoryboardStep` reused it, and the producer phase named a task that never ran. Four fixes, all confined to failure output. **The full run reports product discovery, not capability discovery.** Reusing a failed preparation returned `buildDiscoveryFailedResult` — the generic `discovery_failed` row with task `''`, no validations and `fixture_resolutions` dropped. That sent an implementor to look at their transport, auth and network policy when the actual failure was `get_products`. It now returns a `prepared_fixtures_preparation_failed` step naming the real failing task (read off the producer's own record, not guessed), carrying the `fixture_capture_unbuildable` / `preparation_failed` check, republishing the producer phase with its original context and error, and keeping `fixture_resolutions`. **The standalone run carries the same records.** `runStoryboardStep` returns one step as the entire result, so `StoryboardStepResult` gains an optional `fixture_resolutions` for exactly this case. It had none at all before: the caller was told preparation failed with no way to see which handle, which strategy, or what the seller actually returned. **The producer step names the task that ran.** `chosenStrategy` is only assigned when a strategy RESOLVES, so a discover-only handle that failed left it undefined and the label fell through to `comply_test_controller` — claiming a controller seed for a run whose only traffic was `get_products`. A handle abandoned because its parent failed makes no call at all, so it is named by the strategy it declared rather than by a different tool, and it carries no request record to imply otherwise. **A failed discovery records its real exchange.** The request record is built BEFORE dispatch, so a transport failure still has the request it sent; protocol, schema, shape and pagination failures each cite the page that failed. A malformed page additionally emits a failing `response_schema` check with the schema id and url — the schema grade now runs whether or not the executor accepted the page, because its strict-mode rejection was hiding whether a page was malformed or merely rejected. Nothing is fabricated: a response is recorded only when one arrived, and the `response: null` arm is reserved for an executor that genuinely throws. Unchanged: these stay failures, never `fixture_unsatisfied`; counts and failure meaning are untouched; a denied caller still receives only `PreparedFixturesDeniedError`, with the complete record going to `prepared_fixtures_audit_sink`. Fifteen public regressions cover all four failure types across both entry points, each verified to fail without these changes (1/15 passing before, 15/15 after). Co-Authored-By: Claude Opus 5 (1M context) --- .../fix-3385-failed-preparation-evidence.md | 14 ++ src/lib/testing/storyboard/runner.ts | 125 +++++++++++- src/lib/testing/storyboard/seeding.ts | 179 +++++++++++++--- src/lib/testing/storyboard/types.ts | 10 + test/lib/storyboard-prepared-fixtures.test.js | 191 ++++++++++++++++++ 5 files changed, 481 insertions(+), 38 deletions(-) create mode 100644 .changeset/fix-3385-failed-preparation-evidence.md diff --git a/.changeset/fix-3385-failed-preparation-evidence.md b/.changeset/fix-3385-failed-preparation-evidence.md new file mode 100644 index 000000000..9e4112cce --- /dev/null +++ b/.changeset/fix-3385-failed-preparation-evidence.md @@ -0,0 +1,14 @@ +--- +'@adcp/sdk': patch +--- + +Keep producer evidence and truthful discovery attribution when fixture preparation fails (#3385). + +A `prepareStoryboardFixtures()` that ended in `disposition.failed` lost its evidence the moment an authorized `runStoryboard` / `runStoryboardStep` reused it, and the producer phase named the wrong task. Four fixes, all to failure output; no change to passing runs, counts, or the denial path. + +- **The full run reports product discovery, not capability discovery.** Reusing a failed preparation returned the generic `discovery_failed` row — task `''`, no validations, `fixture_resolutions` dropped — which sent implementors to look at their transport and auth when the actual failure was `get_products`. It now returns a `prepared_fixtures_preparation_failed` step naming the real failing task, carrying the `fixture_capture_unbuildable` / `preparation_failed` check, republishing the producer phase with its original context and error, and keeping `fixture_resolutions`. +- **The standalone run carries the same records.** `runStoryboardStep` returns one step as the whole result, so `StoryboardStepResult` gains an optional `fixture_resolutions` for this case. It had none before: the caller was told preparation failed with no way to see which handle, which strategy, or what the seller returned. +- **The producer step names the task that actually ran.** `chosenStrategy` is only assigned when a strategy RESOLVES, so a discover-only handle that failed fell through to `comply_test_controller` — claiming a controller seed for a run whose only traffic was `get_products`. A handle abandoned because its parent failed makes no call at all and is now named by the strategy it declared, and carries no request record. +- **A failed discovery records its real exchange.** The request is captured before dispatch, so a transport failure still has it; protocol, schema, shape and pagination failures each cite the page that failed. A malformed page also emits a failing `response_schema` check with the schema id and url — the executor's strict-mode rejection used to hide whether a page was malformed or merely rejected. Nothing is fabricated: a response is recorded only when one arrived. + +Unchanged: these remain failures, never `fixture_unsatisfied`; a denied caller still receives only `PreparedFixturesDeniedError`, with the complete record going to `prepared_fixtures_audit_sink`. diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 0afae1134..2d6992064 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -4816,12 +4816,7 @@ async function executeStoryboardPass( const disposition = prepared.state.disposition; if (disposition.status === 'failed') { if (!callerOwnsClients) await closeScopedConnections(options.protocol); - return buildDiscoveryFailedResult(agentUrls, storyboard, { - step: 'Reuse prepared fixture state', - passed: false, - duration_ms: 0, - error: disposition.detail, - }); + return buildPreparationFailedRunResult(agentUrls, storyboard, prepared.state, disposition.detail); } if (disposition.status === 'unsatisfied') { fixtureUnsatisfied = true; @@ -7061,12 +7056,110 @@ function buildPreparedRefusalStepResult( } /** Terminal step result carrying a non-resolved preparation disposition. */ +/** + * Result for a full run reusing a prepared state that FAILED preparation. + * + * Not `buildDiscoveryFailedResult`: that reports agent capability + * discovery, and reusing a failed preparation is a different failure with + * different evidence. Saying `discovery_failed` with task `''` and no + * validations sent an implementor looking at their transport and auth + * when the actual failure was product discovery, and it dropped the + * producer records the preparation had already collected. + * + * The producer phase is republished verbatim so the original context and + * error survive into the consumer's result, and `fixture_resolutions` + * carries the same records `prepared.resolutions` holds. Counts and + * failure meaning are unchanged: this is a failure, never a skip, and + * never `fixture_unsatisfied`. + */ +function buildPreparationFailedRunResult( + agentUrls: string[], + storyboard: Storyboard, + state: PreparedStoryboardFixtures, + detail: string +): StoryboardResult { + const resolutions = state.resolutions; + const producerPhase = state.phase; + // The task that actually failed, read off the producer's own record + // rather than guessed. Absent only when preparation failed before any + // step was recorded. + const failingStep = producerPhase?.steps.find(step => step.passed === false); + const syntheticStep: StoryboardStepResult = { + storyboard_id: storyboard.id, + step_id: 'prepared_fixtures_preparation_failed', + phase_id: 'prepared_fixtures_preparation_failed', + title: 'Storyboard failed: fixture preparation did not resolve', + task: failingStep?.task ?? '', + passed: false, + skipped: false, + duration_ms: 0, + validations: [ + { + check: 'fixture_capture_unbuildable', + passed: false, + description: 'Fixture preparation must resolve before a run can reuse it', + error: detail, + json_pointer: null, + expected: 'a fixture graph that resolved during preparation', + actual: 'preparation_failed', + schema_id: null, + schema_url: null, + // Pre-wire for THIS run: the consumer never dispatched. The + // producer's own exchange is on its step in the phase below. + request: null, + response: null, + }, + ], + context: {}, + extraction: { path: 'none' }, + error: detail, + }; + const phases: StoryboardPhaseResult[] = [ + ...(producerPhase ? [producerPhase] : []), + { + phase_id: 'prepared_fixtures_preparation_failed', + phase_title: 'Prepared fixture preparation failed', + passed: false, + steps: [syntheticStep], + duration_ms: 0, + }, + ]; + return { + storyboard_id: storyboard.id, + storyboard_title: storyboard.title, + agent_url: redactOAuthUrlForOutput(agentUrls[0]!), + agent_urls: agentUrls.map(redactOAuthUrlForOutput), + overall_passed: false, + ...(resolutions.length > 0 && { fixture_resolutions: resolutions }), + ...(state.coverageGaps.length > 0 && { coverage_gaps: state.coverageGaps }), + phases, + context: {}, + total_duration_ms: 0, + passed_count: 0, + failed_count: 1, + skipped_count: 0, + runner_capability_version: RUNNER_CAPABILITY_VERSION, + tested_at: new Date().toISOString(), + strict_validation_summary: { + observable: false, + checked: 0, + passed: 0, + failed: 0, + strict_only_failures: 0, + lenient_also_failed: 0, + }, + notices: [], + }; +} + function buildPreparedDispositionStepResult( storyboard: Storyboard, found: { step: StoryboardStep; phaseId: string }, context: StoryboardContext, disposition: PreparedFixturesDisposition, - contributions: Iterable + contributions: Iterable, + /** Producer records to republish; a standalone result is the whole run. */ + resolutions: readonly FixtureResolutionRecord[] = [] ): StoryboardStepResult { const detail = 'detail' in disposition ? disposition.detail : 'fixture preparation did not resolve'; if (disposition.status === 'failed') { @@ -7096,8 +7189,17 @@ function buildPreparedDispositionStepResult( actual: 'preparation_failed', schema_id: null, schema_url: null, + // Pre-wire for THIS step: the consumer never dispatched. The + // producer's own exchange rides on its records below. + request: null, + response: null, }, ], + // The producer's own records, so a standalone consumer sees the same + // fixture-handle evidence a full run carries. It had none before: + // the caller was told preparation failed with no way to see which + // handle, which strategy, or what the seller actually returned. + ...(resolutions.length > 0 && { fixture_resolutions: [...resolutions] }), context: withoutProtectedCaptures(context), extraction: { path: 'none' }, contributions: Array.from(contributions), @@ -7462,7 +7564,14 @@ async function runStoryboardStepBody( // A terminal preparation outcome is carried forward verbatim. A // retry must not be able to launder a failed graph into a freshly // discovered alternative. - return buildPreparedDispositionStepResult(storyboard, found, context, disposition, contributions); + return buildPreparedDispositionStepResult( + storyboard, + found, + context, + disposition, + contributions, + prepared.state.resolutions + ); } const hydrationViolation = hydratePreparedCaptures(prepared.state, context, protectedWriter, contextProvenance); if (hydrationViolation) { diff --git a/src/lib/testing/storyboard/seeding.ts b/src/lib/testing/storyboard/seeding.ts index 53e65137e..f7e394ea7 100644 --- a/src/lib/testing/storyboard/seeding.ts +++ b/src/lib/testing/storyboard/seeding.ts @@ -593,7 +593,20 @@ function discoveryExchangeFor(catalog: DiscoveryCatalog, productId: string | und type DiscoveryCatalogResult = | { ok: true; catalog: DiscoveryCatalog } - | { ok: false; error: string; evidence?: unknown }; + | { + ok: false; + error: string; + evidence?: unknown; + /** + * The actual exchange that failed, so the producer step can record + * truthful wire evidence instead of nothing. `response` is null only + * when the transport threw and no response was ever received — never + * a stand-in for one we did not capture. + */ + exchange?: { request: RunnerRequestRecord; response: RunnerResponseRecord | null }; + /** AJV detail when the page parsed but failed its response schema. */ + schemaFailure?: string; + }; /** AdCP 3.2 ordered per-handle seed/discover state machine. */ async function runDeclaredFixtureResolution( @@ -774,6 +787,18 @@ async function runDeclaredFixtureResolution( const attempts: FixtureResolutionAttempt[] = []; let boundIds: FixtureResolutionRecord['seller_ids']; let chosenStrategy: FixtureResolutionRecord['strategy']; + /** + * The strategy whose wire call actually ran, successful or not. + * + * `chosenStrategy` is only assigned when a strategy RESOLVES, so a + * discover-only handle that failed left it undefined and the step + * label fell through to `comply_test_controller` — claiming a + * controller seed for a run whose only traffic was `get_products`. + */ + let attemptedStrategy: FixtureResolutionRecord['strategy']; + /** Truthful wire evidence for a failed discovery, never fabricated. */ + let discoveryFailureEvidence: { request: RunnerRequestRecord; response: RunnerResponseRecord | null } | undefined; + let discoverySchemaFailure: string | undefined; let failure: string | undefined; const stepStart = Date.now(); const parentStatus = spec.parentProductHandle ? productStatuses.get(spec.parentProductHandle) : undefined; @@ -790,6 +815,7 @@ async function runDeclaredFixtureResolution( for (const strategy of parentUnavailable ? [] : spec.strategies) { if (strategy === 'seed') { + attemptedStrategy = strategy; await prepareSeedAgent(call); const advertised = controller.scenarios?.has(call.scenario); if (controller.missing || advertised === false) { @@ -899,6 +925,7 @@ async function runDeclaredFixtureResolution( } if (strategy === 'discover') { + attemptedStrategy = strategy; prepareDiscoveryAgent(call); if (discoveryOptions.agentTools && !discoveryOptions.agentTools.includes('get_products')) { attempts.push({ @@ -911,6 +938,8 @@ async function runDeclaredFixtureResolution( const discovered = await catalog(); if (!discovered.ok) { failure = discovered.error; + discoveryFailureEvidence = discovered.exchange; + discoverySchemaFailure = discovered.schemaFailure; attempts.push({ strategy, disposition: 'failed', detail: failure, response: discovered.evidence }); break; } @@ -1000,7 +1029,17 @@ async function runDeclaredFixtureResolution( step_id: call.step_id, phase_id: CONTROLLER_SEEDING_PHASE_ID, title: call.title, - task: chosenStrategy === 'discover' ? 'get_products' : 'comply_test_controller', + // The task that ACTUALLY ran, else the one this handle declared. + // Falling back to the controller whenever no strategy RESOLVED + // attributed a failed discover-only handle to a seed call that + // never happened — and a handle abandoned because its parent + // failed makes no call at all, so it is named by the strategy it + // declared rather than by a different tool. Such a step carries + // no request record, so nothing implies a call was made. + task: + (chosenStrategy ?? attemptedStrategy ?? spec.strategies[0]) === 'discover' + ? 'get_products' + : 'comply_test_controller', passed: status !== 'failed', duration_ms: Date.now() - stepStart, // An unbuildable capture gets a `check` discriminator like the other @@ -1027,9 +1066,35 @@ async function runDeclaredFixtureResolution( ...(captureFailures.get(spec.handle)?.exchange ?? {}), }, ] - : [], + : discoverySchemaFailure !== undefined + ? [ + // A page that parsed but failed its response schema is a + // `response_schema` failure, not a capture failure. Using + // the contract's own kind keeps the schema identity where + // consumers already look for it. + { + check: 'response_schema', + passed: false, + description: 'Fixture discovery response must satisfy the get_products response schema', + error: discoverySchemaFailure, + json_pointer: null, + ...resolveDiscoverySchemaIdentity(options.adcpVersion ?? ADCP_VERSION), + actual: 'schema_validation_failed', + ...(discoveryFailureEvidence && { + request: discoveryFailureEvidence.request, + response: discoveryFailureEvidence.response, + }), + }, + ] + : [], context, extraction: { path: 'none' }, + // The REAL exchange the failing strategy made. Absent rather than + // invented when the strategy never reached the wire. + ...(discoveryFailureEvidence && { + request: discoveryFailureEvidence.request, + ...(discoveryFailureEvidence.response && { response_record: discoveryFailureEvidence.response }), + }), ...(failure && { error: failure }), }); } @@ -1346,6 +1411,40 @@ async function executeLegacySeedCall( } } +/** + * `{ schema_id, schema_url }` for the `get_products` response schema. + * + * Mirrors `resolveSchemaIdentity` in `validations.ts`: the id follows the + * `/schemas//.json` convention cached schemas use for + * `$id`, and the url dereferences against the public docs origin so an + * implementor can fetch exactly what the page was graded against. + */ +function resolveDiscoverySchemaIdentity(adcpVersion: string): { schema_id: string; schema_url: string } { + const schemaId = `/schemas/${adcpVersion}/media-buy/get-products-response.json`; + return { schema_id: schemaId, schema_url: `https://adcontextprotocol.org${schemaId}` }; +} + +/** + * AJV detail for a discovery page that does not satisfy the `get_products` + * response schema, or `undefined` when it does. + */ +function describeDiscoverySchemaFailure(data: unknown, options: StoryboardRunOptions): string | undefined { + try { + const validator = getValidator('get_products', 'sync', options.adcpVersion ?? ADCP_VERSION); + if (validator) { + const schemaData = isPlainRecord(data) ? injectLegacyEnvelopeStatus(data, { toolName: 'get_products' }) : data; + if (validator(schemaData)) return undefined; + return (validator.errors ?? []) + .map(error => `${error.instancePath || '/'} ${error.message ?? error.keyword}`) + .join('; '); + } + const schema = validateResponseSchema('get_products', data, options.adcpVersion); + return schema.passed ? undefined : (schema.error ?? schema.details ?? 'schema validation failed'); + } catch (err) { + return `response schema unavailable: ${err instanceof Error ? err.message : String(err)}`; + } +} + async function discoverProductCatalog( client: TestClient, options: StoryboardRunOptions, @@ -1381,12 +1480,23 @@ async function discoverProductCatalog( }; let result; const pageStart = Date.now(); + // Recorded BEFORE dispatch. A transport failure never reaches the + // response, but the request it sent is real and is the only evidence + // an implementor has to reproduce the failure. + const requestRecord: RunnerRequestRecord = { + transport: 'mcp', + operation: 'get_products', + payload: redactSecrets(request), + }; try { result = await executeStoryboardTask(client, 'get_products', request, { signal: options.signal }); } catch (err) { return { ok: false, error: `fixture discovery transport failure: ${err instanceof Error ? err.message : String(err)}`, + evidence: { page_count: pageCount + 1, request }, + // Explicitly null: no response was received. Not a placeholder. + exchange: { request: requestRecord, response: null }, }; } pageCount++; @@ -1401,7 +1511,7 @@ async function discoverProductCatalog( const retainResponse = exchanges.length < DISCOVERY_RETAINED_RESPONSE_PAGES; const pageIndex = exchanges.push({ - request: { transport: 'mcp', operation: 'get_products', payload: redactSecrets(request) }, + request: requestRecord, response: { transport: 'mcp', payload: retainResponse @@ -1417,55 +1527,54 @@ async function discoverProductCatalog( duration_ms: Date.now() - pageStart, }, }) - 1; + // Graded whether or not the executor accepted the page. A seller that + // returns a malformed body makes the executor reject in strict mode, + // and classifying that as a bare protocol error hid WHICH of the two + // it was — the one thing a `response_schema` record exists to say. + const schemaFailure = result.data === undefined ? undefined : describeDiscoverySchemaFailure(result.data, options); if (!result.success) { return { ok: false, error: `fixture discovery get_products rejected: ${result.error ?? result.adcp_error?.code ?? 'unknown error'}`, evidence: { page_count: pageCount, request }, + exchange: exchanges[pageIndex]!, + ...(schemaFailure !== undefined && { schemaFailure }), }; } - let schemaFailure: string | undefined; - try { - const validator = getValidator('get_products', 'sync', options.adcpVersion ?? ADCP_VERSION); - if (validator) { - const responseData: unknown = result.data; - const schemaData = isPlainRecord(responseData) - ? injectLegacyEnvelopeStatus(responseData, { toolName: 'get_products' }) - : responseData; - if (!validator(schemaData)) { - schemaFailure = (validator.errors ?? []) - .map(error => `${error.instancePath || '/'} ${error.message ?? error.keyword}`) - .join('; '); - } - } else { - const schema = validateResponseSchema('get_products', result.data, options.adcpVersion); - if (!schema.passed) schemaFailure = schema.error ?? schema.details ?? 'schema validation failed'; - } - } catch (err) { - schemaFailure = `response schema unavailable: ${err instanceof Error ? err.message : String(err)}`; - } if (schemaFailure) { return { ok: false, error: `fixture discovery returned malformed get_products response: ${schemaFailure}`, evidence: { page_count: pageCount }, + exchange: exchanges[pageIndex]!, + schemaFailure, }; } if (!isPlainRecord(result.data)) { - return { ok: false, error: 'fixture discovery returned a non-object get_products response' }; + return { + ...{ ok: false, error: 'fixture discovery returned a non-object get_products response' }, + exchange: exchanges[pageIndex]!, + }; } const pageProducts = result.data.products; if (pageProducts !== undefined && !Array.isArray(pageProducts)) { - return { ok: false, error: 'fixture discovery get_products.products is not an array' }; + return { + ...{ ok: false, error: 'fixture discovery get_products.products is not an array' }, + exchange: exchanges[pageIndex]!, + }; } for (const product of pageProducts ?? []) { if (!isPlainRecord(product) || typeof product.product_id !== 'string' || product.product_id.length === 0) { - return { ok: false, error: 'fixture discovery returned a product without a non-empty product_id' }; + return { + ...{ ok: false, error: 'fixture discovery returned a product without a non-empty product_id' }, + exchange: exchanges[pageIndex]!, + }; } if (seenProductIds.has(product.product_id)) { return { ok: false, error: `fixture discovery returned duplicate product_id "${product.product_id}"; response order cannot resolve duplicate identities`, + exchange: exchanges[pageIndex]!, }; } seenProductIds.add(product.product_id); @@ -1477,6 +1586,7 @@ async function discoverProductCatalog( return { ok: false, error: `fixture discovery returned duplicate pricing option identity "${product.product_id}\u0000${option.pricing_option_id}"`, + exchange: exchanges[pageIndex]!, }; } seenPricingIds.add(option.pricing_option_id); @@ -1485,17 +1595,26 @@ async function discoverProductCatalog( pageOfProduct.set(product.product_id, pageIndex); products.push(product); if (products.length > 100_000) { - return { ok: false, error: 'fixture discovery exceeded the 100,000-product safety limit' }; + return { + ...{ ok: false, error: 'fixture discovery exceeded the 100,000-product safety limit' }, + exchange: exchanges[pageIndex]!, + }; } } const pagination = isPlainRecord(result.data.pagination) ? result.data.pagination : undefined; if (pagination?.has_more === true) { const next = pagination.cursor; if (typeof next !== 'string' || next.length === 0) { - return { ok: false, error: 'fixture discovery pagination is broken: has_more=true without cursor' }; + return { + ...{ ok: false, error: 'fixture discovery pagination is broken: has_more=true without cursor' }, + exchange: exchanges[pageIndex]!, + }; } if (seenCursors.has(next)) { - return { ok: false, error: `fixture discovery pagination is broken: repeated cursor "${next}"` }; + return { + ...{ ok: false, error: `fixture discovery pagination is broken: repeated cursor "${next}"` }, + exchange: exchanges[pageIndex]!, + }; } seenCursors.add(next); cursors.push(next); diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index 17eca9a38..e32da8f85 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -2953,6 +2953,16 @@ export interface StoryboardStepResult { agent_url?: string; /** 1-based index of the agent instance (multi-instance mode). Absent in single-URL mode. */ agent_index?: number; + /** + * Producer fixture-handle records, when this step result stands in for a + * whole run. + * + * `runStoryboardStep` returns one step as the entire result, so a + * standalone caller reusing a prepared state has nowhere else to read + * the `fixture_resolutions` block a full run carries. Present only when + * the prepared state actually holds records. + */ + fixture_resolutions?: FixtureResolutionRecord[]; /** Exact request the runner sent (contract-required on failures). */ request?: RunnerRequestRecord; /** Exact response observed, including transport, status, headers. */ diff --git a/test/lib/storyboard-prepared-fixtures.test.js b/test/lib/storyboard-prepared-fixtures.test.js index 25d99ca8d..28cf68cdb 100644 --- a/test/lib/storyboard-prepared-fixtures.test.js +++ b/test/lib/storyboard-prepared-fixtures.test.js @@ -25,6 +25,7 @@ const { prepareStoryboardFixtures, runStoryboard, runStoryboardStep, + withPreparedFixtures, } = require('../../dist/lib/testing/index.js'); const CAPS = { @@ -69,6 +70,35 @@ function product(id, extra = {}) { } function transport(catalog, overrides = {}) { + // `overrides.transportFailure` makes the fetch itself reject, so a test + // can exercise the transport arm without a response ever existing. + if (overrides.transportFailure) { + const wire = []; + const calls = []; + return { + calls, + wire, + options: { + adcpVersion: '3.2.5', + timeout_ms: 5000, + profile: { tools: TOOLS, raw_capabilities: CAPS, supported_protocols: ['media_buy'] }, + transport: { + trustedFetchFn: async (_input, init = {}) => { + const message = JSON.parse(init.body); + wire.push(message.method); + if (message.method === 'tools/call' && message.params?.name === 'get_products') { + throw new Error('synthetic transport failure: connection reset'); + } + return originalTransport(catalog).options.transport.trustedFetchFn(_input, init); + }, + }, + }, + }; + } + return originalTransport(catalog, overrides); +} + +function originalTransport(catalog, overrides = {}) { const calls = []; // Every seller endpoint interaction, including official-client // initialization and tool discovery — not just the mutating calls. A gate @@ -2036,6 +2066,167 @@ describe('refused caller gets a denial; the host gets the evidence', () => { }); }); +describe('failed preparation keeps its producer evidence (#3385)', () => { + /** + * The four ways discovery can fail, each with the shape that produces + * it. Transport never reaches a response; the other three do. + */ + const FAILURE_MODES = { + transport: { transportFailure: true }, + protocol: { + read: () => { + throw new Error('seller rejected get_products'); + }, + }, + schema: { read: () => ({ status: 'completed', products: [{ product_id: 'malformed' }] }) }, + pagination: { + read: () => ({ + status: 'completed', + products: [SELECTED], + cache_scope: 'account', + wholesale_feed_version: 'v1', + // has_more with no cursor: the seller promised another page and + // gave the runner no way to ask for it. + pagination: { has_more: true }, + }), + }, + }; + + /** Prepare against a seller that fails discovery in the named way. */ + async function prepareFailing(mode) { + const io = transport([SELECTED], FAILURE_MODES[mode]); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed', `${mode}: ${JSON.stringify(state.disposition)}`); + assert.ok(state.resolutions.length > 0, `${mode}: the producer recorded its handles`); + return { state, io }; + } + + for (const mode of Object.keys(FAILURE_MODES)) { + test(`${mode}: a full run reuse reports product discovery and keeps the producer records`, async () => { + const { state } = await prepareFailing(mode); + const io = transport([SELECTED]); + const result = await runStoryboard(ENDPOINT, board(), { + ...io.options, + ...withPreparedFixtures(state, ALLOW), + }); + assert.equal(result.overall_passed, false); + assert.equal(result.failed_count, 1); + assert.equal(result.skipped_count, 0, 'a failed preparation is a failure, never a skip'); + + // The producer records survive into the consumer's result. + assert.deepEqual(result.fixture_resolutions, state.resolutions); + + // The failure is named as product discovery, not capability discovery. + const steps = result.phases.flatMap(phase => phase.steps); + const synthetic = steps.find(step => step.step_id === 'prepared_fixtures_preparation_failed'); + assert.ok(synthetic, JSON.stringify(result.phases.map(p => p.phase_id))); + assert.equal(synthetic.task, 'get_products', 'the real failing task, not a blank capability row'); + assert.equal( + steps.some(step => step.step_id === 'discovery_failed'), + false, + 'a failed preparation is not an agent capability-discovery failure' + ); + const check = synthetic.validations.find(v => v.check === 'fixture_capture_unbuildable'); + assert.ok(check, JSON.stringify(synthetic.validations)); + assert.equal(check.actual, 'preparation_failed'); + + // The producer's own phase, with its original context and error. + const producer = result.phases.find(phase => phase.phase_id !== 'prepared_fixtures_preparation_failed'); + assert.ok(producer, 'the producer phase is republished'); + assert.ok( + producer.steps.some(step => step.passed === false), + 'with its failing step intact' + ); + }); + + test(`${mode}: a standalone reuse carries the same producer records`, async () => { + const { state } = await prepareFailing(mode); + const io = transport([SELECTED]); + const result = await runStoryboardStep(ENDPOINT, board(), 'create', { + ...io.options, + ...withPreparedFixtures(state, ALLOW), + }); + assert.equal(result.passed, false); + assert.deepEqual(result.fixture_resolutions, state.resolutions); + const check = result.validations.find(v => v.check === 'fixture_capture_unbuildable'); + assert.ok(check, JSON.stringify(result.validations)); + assert.equal(check.actual, 'preparation_failed'); + assert.equal(check.request, null); + assert.equal(check.response, null); + // No consumer exchange occurred, so none is invented. + assert.equal( + io.calls.some(call => call.task === 'create_media_buy'), + false + ); + }); + + test(`${mode}: the producer step names get_products and never claims a controller seed`, async () => { + const { state } = await prepareFailing(mode); + const producerSteps = state.phase.steps; + const failing = producerSteps.find(step => step.passed === false); + assert.ok(failing, JSON.stringify(producerSteps.map(s => ({ id: s.step_id, passed: s.passed })))); + assert.equal(failing.task, 'get_products', 'the wire call was get_products'); + assert.equal( + producerSteps.some(step => step.task === 'comply_test_controller'), + false, + 'no controller seed happened, so none may be claimed' + ); + }); + } + + test('a failed discovery records the real request, and a response only when one arrived', async () => { + for (const mode of Object.keys(FAILURE_MODES)) { + const { state } = await prepareFailing(mode); + const failing = state.phase.steps.find(step => step.passed === false); + assert.ok(failing.request, `${mode}: the dispatched request is recorded`); + assert.equal(failing.request.operation, 'get_products', mode); + assert.equal(failing.request.payload.buying_mode, 'wholesale', mode); + // A connection-level failure is converted by the task executor into + // an error RESULT rather than a throw, so every one of these four + // arms does come back with something. What is recorded is that real + // payload — the runner never manufactures one. The `response: null` + // arm exists for an executor that genuinely throws. + assert.ok(failing.response_record, `${mode}: the raw response is recorded`); + assert.equal(failing.response_record.transport, 'mcp', mode); + } + }); + + test('a schema failure carries a response_schema check with the schema identity', async () => { + const { state } = await prepareFailing('schema'); + const failing = state.phase.steps.find(step => step.passed === false); + const check = failing.validations.find(v => v.check === 'response_schema'); + assert.ok(check, JSON.stringify(failing.validations)); + assert.equal(check.passed, false); + assert.equal(check.actual, 'schema_validation_failed'); + assert.match(check.schema_id, /get-products-response\.json$/); + assert.match(check.schema_url, /^https:\/\/adcontextprotocol\.org\/schemas\//); + assert.ok(check.request, 'the schema failure cites the request it graded'); + assert.ok(check.response, 'and the response it graded'); + }); + + test('a denied caller still gets only the typed denial, with the record to the sink', async () => { + const { state } = await prepareFailing('schema'); + const io = transport([SELECTED]); + let audited; + await assert.rejects( + () => + runStoryboard(ENDPOINT, board(), { + ...io.options, + prepared_fixtures: state, + prepared_fixtures_audit_sink: record => { + audited = record; + }, + prepared_fixtures_authorizer: () => ({ authorized: false, detail: 'denied' }), + }), + error => error instanceof PreparedFixturesDeniedError && error.refusalCode === 'authorizer_denied' + ); + // A failed preparation does not downgrade the denial: the caller gets + // no producer records, and the host's copy is complete. + assert.ok(audited, 'the sink received the record'); + assert.deepEqual(io.wire, []); + }); +}); + describe('a capture the funnel refuses to detach', () => { /** A storyboard whose product fixture snapshots the whole pinned entity. */ function snapshotBoard() { From 9a59a312a798276dae30325000e60ea40772f7ab Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 24 Oct 2026 15:11:18 +0000 Subject: [PATCH 2/2] fix(testing): preserve truthful failed preparation diagnostics --- .../fix-3385-failed-preparation-evidence.md | 11 +- docs/guides/COLLECTION-FIXTURE-FOUNDATION.md | 11 + src/lib/protocols/mcp.ts | 2 + src/lib/protocols/rawResponseCapture.ts | 26 +- src/lib/testing/storyboard/runner.ts | 44 ++- src/lib/testing/storyboard/seeding.ts | 231 ++++++++++--- src/lib/testing/storyboard/task-map.ts | 3 +- src/lib/testing/storyboard/types.ts | 4 + test/lib/mcp-oauth-scoped-fetch.test.js | 3 + test/lib/raw-response-capture.test.js | 22 ++ test/lib/storyboard-prepared-fixtures.test.js | 306 +++++++++++++++++- 11 files changed, 568 insertions(+), 95 deletions(-) diff --git a/.changeset/fix-3385-failed-preparation-evidence.md b/.changeset/fix-3385-failed-preparation-evidence.md index 9e4112cce..d0865f83d 100644 --- a/.changeset/fix-3385-failed-preparation-evidence.md +++ b/.changeset/fix-3385-failed-preparation-evidence.md @@ -2,13 +2,8 @@ '@adcp/sdk': patch --- -Keep producer evidence and truthful discovery attribution when fixture preparation fails (#3385). +Preserve producer evidence when prepared fixture discovery fails (#3385). -A `prepareStoryboardFixtures()` that ended in `disposition.failed` lost its evidence the moment an authorized `runStoryboard` / `runStoryboardStep` reused it, and the producer phase named the wrong task. Four fixes, all to failure output; no change to passing runs, counts, or the denial path. +Authorized full and standalone reuse now retain fixture-handle resolutions and a `fixture_preparation` diagnostic phase with the producer's errors and actual exchanges. The historical phase is outside graded `phases`, preserving consumer and JUnit failure counts. Protected captures are stripped from its public contexts; denied callers still receive only `PreparedFixturesDeniedError`, while the private audit sink retains failed producer evidence. -- **The full run reports product discovery, not capability discovery.** Reusing a failed preparation returned the generic `discovery_failed` row — task `''`, no validations, `fixture_resolutions` dropped — which sent implementors to look at their transport and auth when the actual failure was `get_products`. It now returns a `prepared_fixtures_preparation_failed` step naming the real failing task, carrying the `fixture_capture_unbuildable` / `preparation_failed` check, republishing the producer phase with its original context and error, and keeping `fixture_resolutions`. -- **The standalone run carries the same records.** `runStoryboardStep` returns one step as the whole result, so `StoryboardStepResult` gains an optional `fixture_resolutions` for this case. It had none before: the caller was told preparation failed with no way to see which handle, which strategy, or what the seller returned. -- **The producer step names the task that actually ran.** `chosenStrategy` is only assigned when a strategy RESOLVES, so a discover-only handle that failed fell through to `comply_test_controller` — claiming a controller seed for a run whose only traffic was `get_products`. A handle abandoned because its parent failed makes no call at all and is now named by the strategy it declared, and carries no request record. -- **A failed discovery records its real exchange.** The request is captured before dispatch, so a transport failure still has it; protocol, schema, shape and pagination failures each cite the page that failed. A malformed page also emits a failing `response_schema` check with the schema id and url — the executor's strict-mode rejection used to hide whether a page was malformed or merely rejected. Nothing is fabricated: a response is recorded only when one arrived. - -Unchanged: these remain failures, never `fixture_unsatisfied`; a denied caller still receives only `PreparedFixturesDeniedError`, with the complete record going to `prepared_fixtures_audit_sink`. +Discovery failures identify the attempted task and configured transport. SDK connection errors are never represented as seller responses, protocol rejections remain protocol failures, and malformed seller pages retain a schema diagnostic against the actual received payload before SDK projection. The terminal failed page is retained even beyond the bounded successful-page evidence history. Captured text payloads redact credential fields. diff --git a/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md b/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md index ac151a917..df1c2fe9b 100644 --- a/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md +++ b/docs/guides/COLLECTION-FIXTURE-FOUNDATION.md @@ -717,6 +717,17 @@ Until a companion `adcontextprotocol/adcp` PR lands all six, no *published* storyboard can legally use any of this. That is a merge dependency, not an implementation gap — and it is why the downgrade story below matters. +When a prepared fixture graph failed discovery, authorized reuse retains its +producer phase as `fixture_preparation` and its handle records as +`fixture_resolutions`. The producer phase is diagnostic history outside graded +`phases`, so it does not add JUnit failures or change consumer grading counts. +Protected capture keys are stripped from its public step contexts; a private +audit sink receives the complete failed producer record after a denial. Failed +discovery responses retain the official client's protocol response when available; +successful-page evidence retains parsed SDK data. HTTP-only A2A evidence includes +its status and an extraction note when the bounded capture was truncated. SDK +connection diagnostics never stand in for a seller response. + ### Output inventory for the amendment Everything a report consumer can newly observe, with where it is emitted and diff --git a/src/lib/protocols/mcp.ts b/src/lib/protocols/mcp.ts index 3fd60da24..b1e81d7ac 100644 --- a/src/lib/protocols/mcp.ts +++ b/src/lib/protocols/mcp.ts @@ -1450,6 +1450,7 @@ export async function callMCPToolWithOAuth(options: MCPCallOptions): Promise { - let text: string; + if (!response.body) return { body: '', bodyTruncated: false }; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ''; + let complete = false; try { - text = await response.text(); + while (true) { + const chunk = await reader.read(); + if (chunk.done) { + text += decoder.decode(); + complete = true; + return { body: text.slice(0, maxBodyBytes), bodyTruncated: text.length > maxBodyBytes }; + } + text += decoder.decode(chunk.value, { stream: true }); + if (text.length > maxBodyBytes) return { body: text.slice(0, maxBodyBytes), bodyTruncated: true }; + } } catch { - return { body: '', bodyTruncated: false }; + return { body: text, bodyTruncated: true }; + } finally { + // A cloned response is a tee. Waiting for cancellation here can deadlock + // until the original body is consumed, which happens only after fetch returns. + if (!complete) void reader.cancel().catch(() => {}); + reader.releaseLock(); } - if (text.length <= maxBodyBytes) return { body: text, bodyTruncated: false }; - return { body: text.slice(0, maxBodyBytes), bodyTruncated: true }; } diff --git a/src/lib/testing/storyboard/runner.ts b/src/lib/testing/storyboard/runner.ts index 2d6992064..4d7869bf0 100644 --- a/src/lib/testing/storyboard/runner.ts +++ b/src/lib/testing/storyboard/runner.ts @@ -53,6 +53,7 @@ import { preparedBindingRegistryFor, PreparedStoryboardFixtures, resolvePreparedFixtures, + isTrustedPreparedFixtures, type PreparedFixtureBinding, type PreparedFixturesDisposition, type PreparedFixturesRefusal, @@ -6951,7 +6952,16 @@ async function emitPreparedFixturesAudit( const sink = options.prepared_fixtures_audit_sink; if (!sink) return; try { - await sink(record); + const state = options.prepared_fixtures; + const auditRecord = + isTrustedPreparedFixtures(state) && state.disposition.status === 'failed' + ? { + ...record, + fixture_resolutions: state.resolutions, + ...(state.phase && { fixture_preparation: state.phase }), + } + : record; + await sink(auditRecord); } catch { // Intentionally swallowed; see above. } @@ -7055,7 +7065,11 @@ function buildPreparedRefusalStepResult( }; } -/** Terminal step result carrying a non-resolved preparation disposition. */ +/** Historical producer evidence does not add graded steps to the consumer run. */ +function preparedPhaseForOutput(phase: StoryboardPhaseResult): StoryboardPhaseResult { + return { ...phase, steps: phase.steps.map(step => ({ ...step, context: withoutProtectedCaptures(step.context) })) }; +} + /** * Result for a full run reusing a prepared state that FAILED preparation. * @@ -7066,8 +7080,9 @@ function buildPreparedRefusalStepResult( * when the actual failure was product discovery, and it dropped the * producer records the preparation had already collected. * - * The producer phase is republished verbatim so the original context and - * error survive into the consumer's result, and `fixture_resolutions` + * The producer phase is retained separately as diagnostic evidence, with + * protected captures stripped from context. Its errors and exchanges survive, + * and `fixture_resolutions` * carries the same records `prepared.resolutions` holds. Counts and * failure meaning are unchanged: this is a failure, never a skip, and * never `fixture_unsatisfied`. @@ -7105,7 +7120,7 @@ function buildPreparationFailedRunResult( schema_id: null, schema_url: null, // Pre-wire for THIS run: the consumer never dispatched. The - // producer's own exchange is on its step in the phase below. + // producer's own exchange is retained in fixture_preparation. request: null, response: null, }, @@ -7115,7 +7130,6 @@ function buildPreparationFailedRunResult( error: detail, }; const phases: StoryboardPhaseResult[] = [ - ...(producerPhase ? [producerPhase] : []), { phase_id: 'prepared_fixtures_preparation_failed', phase_title: 'Prepared fixture preparation failed', @@ -7133,6 +7147,7 @@ function buildPreparationFailedRunResult( ...(resolutions.length > 0 && { fixture_resolutions: resolutions }), ...(state.coverageGaps.length > 0 && { coverage_gaps: state.coverageGaps }), phases, + ...(producerPhase && { fixture_preparation: preparedPhaseForOutput(producerPhase) }), context: {}, total_duration_ms: 0, passed_count: 0, @@ -7140,18 +7155,12 @@ function buildPreparationFailedRunResult( skipped_count: 0, runner_capability_version: RUNNER_CAPABILITY_VERSION, tested_at: new Date().toISOString(), - strict_validation_summary: { - observable: false, - checked: 0, - passed: 0, - failed: 0, - strict_only_failures: 0, - lenient_also_failed: 0, - }, + strict_validation_summary: summarizeStrictValidation(phases), notices: [], }; } +/** Terminal step result carrying a non-resolved preparation disposition. */ function buildPreparedDispositionStepResult( storyboard: Storyboard, found: { step: StoryboardStep; phaseId: string }, @@ -7159,7 +7168,8 @@ function buildPreparedDispositionStepResult( disposition: PreparedFixturesDisposition, contributions: Iterable, /** Producer records to republish; a standalone result is the whole run. */ - resolutions: readonly FixtureResolutionRecord[] = [] + resolutions: readonly FixtureResolutionRecord[] = [], + producerPhase?: StoryboardPhaseResult ): StoryboardStepResult { const detail = 'detail' in disposition ? disposition.detail : 'fixture preparation did not resolve'; if (disposition.status === 'failed') { @@ -7200,6 +7210,7 @@ function buildPreparedDispositionStepResult( // the caller was told preparation failed with no way to see which // handle, which strategy, or what the seller actually returned. ...(resolutions.length > 0 && { fixture_resolutions: [...resolutions] }), + ...(producerPhase && { fixture_preparation: preparedPhaseForOutput(producerPhase) }), context: withoutProtectedCaptures(context), extraction: { path: 'none' }, contributions: Array.from(contributions), @@ -7570,7 +7581,8 @@ async function runStoryboardStepBody( context, disposition, contributions, - prepared.state.resolutions + prepared.state.resolutions, + prepared.state.phase ); } const hydrationViolation = hydratePreparedCaptures(prepared.state, context, protectedWriter, contextProvenance); diff --git a/src/lib/testing/storyboard/seeding.ts b/src/lib/testing/storyboard/seeding.ts index f7e394ea7..d7262236b 100644 --- a/src/lib/testing/storyboard/seeding.ts +++ b/src/lib/testing/storyboard/seeding.ts @@ -25,8 +25,10 @@ import { resolveAccount, resolveBrand, validateResponseSchema, type TestClient } import { getValidator } from '../../validation/schema-loader'; import { ADCP_VERSION } from '../../version'; import { injectLegacyEnvelopeStatus } from '../../utils/envelope-status-compat'; +import { readExtractionPath, unwrapProtocolResponse } from '../../utils/response-unwrapper'; import { callControllerRaw } from '../test-controller'; import { executeStoryboardTask } from './task-map'; +import { getCapturesFromError, withRawResponseCapture, type RawHttpCapture } from '../../protocols/rawResponseCapture'; import { buildFixtureResolutionSpecs, applyFixtureBindingsToRequest, @@ -37,10 +39,13 @@ import { } from './fixture-resolution'; import { readCollectionWitness, type CollectionSelectionWitness } from './collection-selection'; import { resolvePath } from './path'; -import { redactSecrets } from '../../utils/redact-secrets'; +import { redactSecrets, SECRET_KEY_PATTERN } from '../../utils/redact-secrets'; +import { redactCredentialPatterns } from '../../utils/redact-credential-patterns'; +import { MAX_JSON_DEPTH } from '../../utils/json-depth'; import type { ContextProvenanceEntry, FixtureResolutionAttempt, + RunnerExtractionRecord, RunnerRequestRecord, RunnerResponseRecord, FixtureResolutionRecord, @@ -603,9 +608,13 @@ type DiscoveryCatalogResult = * when the transport threw and no response was ever received — never * a stand-in for one we did not capture. */ - exchange?: { request: RunnerRequestRecord; response: RunnerResponseRecord | null }; + exchange?: { + request: RunnerRequestRecord; + response: RunnerResponseRecord | null; + extraction?: RunnerExtractionRecord; + }; /** AJV detail when the page parsed but failed its response schema. */ - schemaFailure?: string; + schemaFailure?: { detail: string; schema_id: string; schema_url: string }; }; /** AdCP 3.2 ordered per-handle seed/discover state machine. */ @@ -797,8 +806,10 @@ async function runDeclaredFixtureResolution( */ let attemptedStrategy: FixtureResolutionRecord['strategy']; /** Truthful wire evidence for a failed discovery, never fabricated. */ - let discoveryFailureEvidence: { request: RunnerRequestRecord; response: RunnerResponseRecord | null } | undefined; - let discoverySchemaFailure: string | undefined; + let discoveryFailureEvidence: + | { request: RunnerRequestRecord; response: RunnerResponseRecord | null; extraction?: RunnerExtractionRecord } + | undefined; + let discoverySchemaFailure: Extract['schemaFailure']; let failure: string | undefined; const stepStart = Date.now(); const parentStatus = spec.parentProductHandle ? productStatuses.get(spec.parentProductHandle) : undefined; @@ -815,7 +826,6 @@ async function runDeclaredFixtureResolution( for (const strategy of parentUnavailable ? [] : spec.strategies) { if (strategy === 'seed') { - attemptedStrategy = strategy; await prepareSeedAgent(call); const advertised = controller.scenarios?.has(call.scenario); if (controller.missing || advertised === false) { @@ -828,6 +838,7 @@ async function runDeclaredFixtureResolution( }); continue; } + attemptedStrategy = strategy; try { const controllerRequest = applyFixtureBindingsToRequest( { scenario: call.scenario, params: call.params }, @@ -925,7 +936,6 @@ async function runDeclaredFixtureResolution( } if (strategy === 'discover') { - attemptedStrategy = strategy; prepareDiscoveryAgent(call); if (discoveryOptions.agentTools && !discoveryOptions.agentTools.includes('get_products')) { attempts.push({ @@ -935,6 +945,7 @@ async function runDeclaredFixtureResolution( }); continue; } + attemptedStrategy = strategy; const discovered = await catalog(); if (!discovered.ok) { failure = discovered.error; @@ -1076,9 +1087,11 @@ async function runDeclaredFixtureResolution( check: 'response_schema', passed: false, description: 'Fixture discovery response must satisfy the get_products response schema', - error: discoverySchemaFailure, + error: discoverySchemaFailure.detail, json_pointer: null, - ...resolveDiscoverySchemaIdentity(options.adcpVersion ?? ADCP_VERSION), + schema_id: discoverySchemaFailure.schema_id, + schema_url: discoverySchemaFailure.schema_url, + expected: discoverySchemaFailure.schema_id, actual: 'schema_validation_failed', ...(discoveryFailureEvidence && { request: discoveryFailureEvidence.request, @@ -1088,7 +1101,7 @@ async function runDeclaredFixtureResolution( ] : [], context, - extraction: { path: 'none' }, + extraction: discoveryFailureEvidence?.extraction ?? { path: 'none' }, // The REAL exchange the failing strategy made. Absent rather than // invented when the strategy never reached the wire. ...(discoveryFailureEvidence && { @@ -1424,24 +1437,53 @@ function resolveDiscoverySchemaIdentity(adcpVersion: string): { schema_id: strin return { schema_id: schemaId, schema_url: `https://adcontextprotocol.org${schemaId}` }; } +/** Preserve protocol evidence while also scrubbing credentials inside text payloads. */ +function redactDiscoveryResponse(value: unknown): unknown { + const visit = (node: unknown, depth: number): unknown => { + if (depth > MAX_JSON_DEPTH) return '[redacted]'; + if (typeof node === 'string') { + return (redactCredentialPatterns(node, false) as string).replaceAll('', '[redacted]'); + } + if (Array.isArray(node)) return node.map(child => visit(child, depth + 1)); + if (node && typeof node === 'object') { + return Object.fromEntries(Object.entries(node).map(([key, child]) => [key, visit(child, depth + 1)])); + } + return node; + }; + const pattern = new RegExp( + `${SECRET_KEY_PATTERN.source}|^(private[_-]?key|signing[_-]?key|signature|auth[_-]?token|jwt)$`, + 'i' + ); + return visit(redactSecrets(value, pattern), 0); +} + /** * AJV detail for a discovery page that does not satisfy the `get_products` * response schema, or `undefined` when it does. */ -function describeDiscoverySchemaFailure(data: unknown, options: StoryboardRunOptions): string | undefined { +function describeDiscoverySchemaFailure( + data: unknown, + options: StoryboardRunOptions +): { error: string; schemaFailure?: { detail: string; schema_id: string; schema_url: string } } | undefined { try { - const validator = getValidator('get_products', 'sync', options.adcpVersion ?? ADCP_VERSION); + const version = options.adcpVersion ?? ADCP_VERSION; + const validator = getValidator('get_products', 'sync', version); + let detail: string; if (validator) { const schemaData = isPlainRecord(data) ? injectLegacyEnvelopeStatus(data, { toolName: 'get_products' }) : data; if (validator(schemaData)) return undefined; - return (validator.errors ?? []) + detail = (validator.errors ?? []) .map(error => `${error.instancePath || '/'} ${error.message ?? error.keyword}`) .join('; '); + } else { + const schema = validateResponseSchema('get_products', data, options.adcpVersion); + if (schema.passed) return undefined; + detail = schema.error ?? schema.details ?? 'schema validation failed'; } - const schema = validateResponseSchema('get_products', data, options.adcpVersion); - return schema.passed ? undefined : (schema.error ?? schema.details ?? 'schema validation failed'); + return { error: detail, schemaFailure: { detail, ...resolveDiscoverySchemaIdentity(version) } }; } catch (err) { - return `response schema unavailable: ${err instanceof Error ? err.message : String(err)}`; + // Missing runner schemas are a setup failure, not evidence against the seller. + return { error: `response schema unavailable: ${err instanceof Error ? err.message : String(err)}` }; } } @@ -1479,27 +1521,100 @@ async function discoverProductCatalog( ext: { adcp: { creative_wire: 'canonical' } }, }; let result; + let observedResponse: RawHttpCapture | undefined; + const toolResponse = (captures: RawHttpCapture[]) => + [...captures].reverse().find(capture => capture.requestAdcpSkill === 'get_products'); const pageStart = Date.now(); // Recorded BEFORE dispatch. A transport failure never reaches the // response, but the request it sent is real and is the only evidence // an implementor has to reproduce the failure. const requestRecord: RunnerRequestRecord = { - transport: 'mcp', + transport: options.protocol === 'a2a' ? 'a2a' : 'mcp', operation: 'get_products', payload: redactSecrets(request), }; try { - result = await executeStoryboardTask(client, 'get_products', request, { signal: options.signal }); + // MCP's official SDK trace already supplies its parsed protocol response. + // Capturing its HTTP streams would buffer an unrelated SSE subscription. + if (options.protocol === 'a2a') { + const captured = await withRawResponseCapture( + () => executeStoryboardTask(client, 'get_products', request, { signal: options.signal }), + { maxBodyBytes: 65_536, maxCaptures: 8, maxTotalBodyBytes: 262_144 } + ); + result = captured.result; + observedResponse = toolResponse(captured.captures); + } else { + result = await executeStoryboardTask(client, 'get_products', request, { signal: options.signal }); + } } catch (err) { + const observed = toolResponse(getCapturesFromError(err) ?? []); return { ok: false, - error: `fixture discovery transport failure: ${err instanceof Error ? err.message : String(err)}`, - evidence: { page_count: pageCount + 1, request }, + error: `fixture discovery transport failure: ${redactCredentialPatterns(err instanceof Error ? err.message : String(err), false)}`, + evidence: { page_count: pageCount + 1, request: requestRecord.payload }, // Explicitly null: no response was received. Not a placeholder. - exchange: { request: requestRecord, response: null }, + exchange: { + request: requestRecord, + response: observed + ? { + transport: requestRecord.transport, + payload: redactDiscoveryResponse(observed.body), + status: observed.status, + duration_ms: Date.now() - pageStart, + } + : null, + }, }; } pageCount++; + // The official clients retain the received protocol response before SDK + // validation/projection. A connection error has no such response entry. + let wireResponse = result.debug_logs + ?.slice() + .reverse() + .find( + log => + typeof log.message === 'string' && + (log.message.startsWith('MCP: Tool get_products response received') || + log.message.startsWith('A2A: Response received')) && + log.response !== undefined + )?.response; + if (wireResponse === undefined && observedResponse && !observedResponse.bodyTruncated) { + try { + wireResponse = JSON.parse(observedResponse.body); + } catch { + // Raw diagnostics remain available; protocol decoding belongs to the SDK. + } + } + // An executor error is SDK diagnostic text, not a seller response. Retain + // a response only when a tool body or actual HTTP exchange was observed. + if ( + (!result.success || result.data === undefined) && + result._extraction_path === undefined && + observedResponse === undefined && + wireResponse === undefined + ) { + return { + ok: false, + error: `fixture discovery get_products rejected: ${redactCredentialPatterns(result.error ?? result.adcp_error?.code ?? 'unknown error', false)}`, + evidence: { page_count: pageCount, request: requestRecord.payload }, + exchange: { request: requestRecord, response: null }, + }; + } + const extraction: RunnerExtractionRecord = { + path: result._extraction_path ?? 'none', + ...(observedResponse?.bodyTruncated && { note: 'Raw HTTP response capture was truncated at its body budget.' }), + }; + const failedExchange = { + request: requestRecord, + extraction, + response: { + transport: requestRecord.transport, + payload: redactDiscoveryResponse(wireResponse ?? observedResponse?.body ?? result.data), + ...(observedResponse && { status: observedResponse.status }), + duration_ms: Date.now() - pageStart, + }, + }; // Recorded for EVERY page, success or not: a schema-invalid or rejected // page is exactly the exchange a reader needs to see. // The request is tiny and always retained. The RESPONSE is a second @@ -1512,69 +1627,85 @@ async function discoverProductCatalog( const pageIndex = exchanges.push({ request: requestRecord, - response: { - transport: 'mcp', - payload: retainResponse - ? redactSecrets( - result.data ?? - (result.adcp_error ? { adcp_error: result.adcp_error } : undefined) ?? - result.error ?? - null - ) - : { + response: retainResponse + ? { ...failedExchange.response, payload: redactSecrets(result.data) } + : { + transport: requestRecord.transport, + payload: { response_not_retained: `page ${exchanges.length + 1} is past the runner's ${DISCOVERY_RETAINED_RESPONSE_PAGES}-page discovery-evidence budget; the request for this page is recorded in full`, }, - duration_ms: Date.now() - pageStart, - }, + duration_ms: Date.now() - pageStart, + }, }) - 1; // Graded whether or not the executor accepted the page. A seller that // returns a malformed body makes the executor reject in strict mode, // and classifying that as a bare protocol error hid WHICH of the two // it was — the one thing a `response_schema` record exists to say. - const schemaFailure = result.data === undefined ? undefined : describeDiscoverySchemaFailure(result.data, options); + let schemaData = result.success ? result.data : undefined; + if (!result.success && wireResponse !== undefined && result._extraction_path !== 'error') { + try { + schemaData = unwrapProtocolResponse(wireResponse, undefined, options.protocol === 'a2a' ? 'a2a' : 'mcp'); + extraction.path = readExtractionPath(schemaData) ?? extraction.path; + } catch { + // Protocol decoding errors retain their original classification. + schemaData = undefined; + } + } + const nativeResult = + isPlainRecord(wireResponse) && isPlainRecord(wireResponse.result) ? wireResponse.result : undefined; + const nativeState = nativeResult && isPlainRecord(nativeResult.status) ? nativeResult.status.state : undefined; + const sellerRejected = + result._extraction_path === 'error' || + (isPlainRecord(schemaData) && + (schemaData.adcp_error !== undefined || schemaData.status === 'failed' || schemaData.status === 'rejected')) || + nativeState === 'failed' || + nativeState === 'rejected' || + nativeState === 'canceled'; + const schemaCheck = + sellerRejected || schemaData === undefined ? undefined : describeDiscoverySchemaFailure(schemaData, options); if (!result.success) { return { ok: false, - error: `fixture discovery get_products rejected: ${result.error ?? result.adcp_error?.code ?? 'unknown error'}`, - evidence: { page_count: pageCount, request }, - exchange: exchanges[pageIndex]!, - ...(schemaFailure !== undefined && { schemaFailure }), + error: `fixture discovery get_products rejected: ${redactCredentialPatterns(result.error ?? result.adcp_error?.code ?? 'unknown error', false)}`, + evidence: { page_count: pageCount, request: requestRecord.payload }, + exchange: failedExchange, + ...(schemaCheck?.schemaFailure && { schemaFailure: schemaCheck.schemaFailure }), }; } - if (schemaFailure) { + if (schemaCheck) { return { ok: false, - error: `fixture discovery returned malformed get_products response: ${schemaFailure}`, + error: `fixture discovery get_products validation failed: ${schemaCheck.error}`, evidence: { page_count: pageCount }, - exchange: exchanges[pageIndex]!, - schemaFailure, + exchange: failedExchange, + ...(schemaCheck.schemaFailure && { schemaFailure: schemaCheck.schemaFailure }), }; } if (!isPlainRecord(result.data)) { return { ...{ ok: false, error: 'fixture discovery returned a non-object get_products response' }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } const pageProducts = result.data.products; if (pageProducts !== undefined && !Array.isArray(pageProducts)) { return { ...{ ok: false, error: 'fixture discovery get_products.products is not an array' }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } for (const product of pageProducts ?? []) { if (!isPlainRecord(product) || typeof product.product_id !== 'string' || product.product_id.length === 0) { return { ...{ ok: false, error: 'fixture discovery returned a product without a non-empty product_id' }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } if (seenProductIds.has(product.product_id)) { return { ok: false, error: `fixture discovery returned duplicate product_id "${product.product_id}"; response order cannot resolve duplicate identities`, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } seenProductIds.add(product.product_id); @@ -1586,7 +1717,7 @@ async function discoverProductCatalog( return { ok: false, error: `fixture discovery returned duplicate pricing option identity "${product.product_id}\u0000${option.pricing_option_id}"`, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } seenPricingIds.add(option.pricing_option_id); @@ -1597,7 +1728,7 @@ async function discoverProductCatalog( if (products.length > 100_000) { return { ...{ ok: false, error: 'fixture discovery exceeded the 100,000-product safety limit' }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } } @@ -1607,13 +1738,13 @@ async function discoverProductCatalog( if (typeof next !== 'string' || next.length === 0) { return { ...{ ok: false, error: 'fixture discovery pagination is broken: has_more=true without cursor' }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } if (seenCursors.has(next)) { return { ...{ ok: false, error: `fixture discovery pagination is broken: repeated cursor "${next}"` }, - exchange: exchanges[pageIndex]!, + exchange: failedExchange, }; } seenCursors.add(next); diff --git a/src/lib/testing/storyboard/task-map.ts b/src/lib/testing/storyboard/task-map.ts index 1eb411457..2039551e4 100644 --- a/src/lib/testing/storyboard/task-map.ts +++ b/src/lib/testing/storyboard/task-map.ts @@ -8,6 +8,7 @@ */ import type { TaskResult } from '../types'; +import type { DebugLogEntry } from '../../types/adcp'; import type { AdcpErrorInfo } from '../../core/ConversationTypes'; import { isTerminalAdcpError, readExtractionPath } from '../../utils/response-unwrapper'; import { generateIdempotencyKey } from '../../utils/idempotency'; @@ -316,7 +317,7 @@ export async function executeStoryboardTask( taskName: string, params: Record, opts: StoryboardTaskExecutionOptions = {} -): Promise { +): Promise { const unresolvedAssetDirectives = findUnresolvedCreativeAssetDirectives(params); if (unresolvedAssetDirectives.length > 0) { return { diff --git a/src/lib/testing/storyboard/types.ts b/src/lib/testing/storyboard/types.ts index e32da8f85..5059f4d00 100644 --- a/src/lib/testing/storyboard/types.ts +++ b/src/lib/testing/storyboard/types.ts @@ -2963,6 +2963,8 @@ export interface StoryboardStepResult { * the prepared state actually holds records. */ fixture_resolutions?: FixtureResolutionRecord[]; + /** Historical producer evidence; excluded from consumer grading and JUnit counts. */ + fixture_preparation?: StoryboardPhaseResult; /** Exact request the runner sent (contract-required on failures). */ request?: RunnerRequestRecord; /** Exact response observed, including transport, status, headers. */ @@ -3456,6 +3458,8 @@ export interface StoryboardResult { overall_passed: boolean; /** AdCP 3.2 fixture-handle setup/discovery evidence, one row per handle. */ fixture_resolutions?: FixtureResolutionRecord[]; + /** Historical producer evidence; excluded from consumer grading and JUnit counts. */ + fixture_preparation?: StoryboardPhaseResult; /** Storyboard-level fixture coverage gaps, emitted once per cause. */ coverage_gaps?: FixtureResolutionCoverageGap[]; /** diff --git a/test/lib/mcp-oauth-scoped-fetch.test.js b/test/lib/mcp-oauth-scoped-fetch.test.js index f9ddcba8c..5c9ddeddf 100644 --- a/test/lib/mcp-oauth-scoped-fetch.test.js +++ b/test/lib/mcp-oauth-scoped-fetch.test.js @@ -576,14 +576,17 @@ test('modern OAuth refresh uses the scoped fetcher instead of global fetch', asy const server = await startOAuthServer('modern'); try { await withGlobalFetchGuard(async (fetchFn, fetchedUrls) => { + const debugLogs = []; const result = await callMCPToolWithOAuth({ agentUrl: server.url, toolName: 'ping', args: {}, authProvider: createRefreshProvider(server.state.origin), fetchFn, + debugLogs, }); assert.equal(result.content[0].text, 'pong'); + assert.deepEqual(debugLogs.find(log => log.message === 'MCP: Tool ping response received').response, result); assert.ok( fetchedUrls.some(url => url.endsWith('/token')), 'scoped fetcher should perform the token exchange' diff --git a/test/lib/raw-response-capture.test.js b/test/lib/raw-response-capture.test.js index 84c1da254..8aaf30169 100644 --- a/test/lib/raw-response-capture.test.js +++ b/test/lib/raw-response-capture.test.js @@ -149,6 +149,28 @@ describe('rawResponseCapture', () => { assert.doesNotMatch(JSON.stringify(captures[0]), /must-not-be-captured/); }); + test('bounded capture returns without waiting for an oversized stream to finish', { timeout: 2000 }, async t => { + let controller; + const stream = new ReadableStream({ + start(value) { + controller = value; + value.enqueue(new TextEncoder().encode('x'.repeat(1024))); + }, + }); + t.after(() => { + try { + controller.close(); + } catch {} + }); + const capturingFetch = wrapFetchWithCapture(async () => new Response(stream)); + const { result, captures } = await withRawResponseCapture(() => capturingFetch('https://example.test/'), { + maxBodyBytes: 32, + }); + assert.equal(captures[0].body.length, 32); + assert.equal(captures[0].bodyTruncated, true); + await result.body.cancel(); + }); + test('truncates body when it exceeds maxBodyBytes', async () => { const big = 'A'.repeat(10_000); const { server, url } = await startServer((req, res) => { diff --git a/test/lib/storyboard-prepared-fixtures.test.js b/test/lib/storyboard-prepared-fixtures.test.js index 28cf68cdb..6074c78f1 100644 --- a/test/lib/storyboard-prepared-fixtures.test.js +++ b/test/lib/storyboard-prepared-fixtures.test.js @@ -128,6 +128,33 @@ function originalTransport(catalog, overrides = {}) { } else if (message.method === 'tools/call') { const { name, arguments: request } = message.params; calls.push({ task: name, request }); + if (name === 'get_products' && overrides.protocolFailure) { + return new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { + isError: true, + content: [ + { + type: 'text', + text: JSON.stringify( + overrides.protocolPayload ?? { + status: 'failed', + adcp_error: { + code: 'INVALID_REQUEST', + message: 'Seller refused discovery', + recovery: 'terminal', + }, + } + ), + }, + ], + }, + }), + { headers: { 'content-type': 'application/json' } } + ); + } let data; if (name === 'get_adcp_capabilities') data = CAPS; else if (name === 'get_products') @@ -137,6 +164,7 @@ function originalTransport(catalog, overrides = {}) { cache_scope: 'account', wholesale_feed_version: 'v1', }; + else if (name === 'comply_test_controller' && overrides.controller) data = overrides.controller(request); else if (name === 'create_media_buy') data = { status: 'completed', @@ -2073,11 +2101,7 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { */ const FAILURE_MODES = { transport: { transportFailure: true }, - protocol: { - read: () => { - throw new Error('seller rejected get_products'); - }, - }, + protocol: { protocolFailure: true }, schema: { read: () => ({ status: 'completed', products: [{ product_id: 'malformed' }] }) }, pagination: { read: () => ({ @@ -2131,12 +2155,22 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { assert.equal(check.actual, 'preparation_failed'); // The producer's own phase, with its original context and error. - const producer = result.phases.find(phase => phase.phase_id !== 'prepared_fixtures_preparation_failed'); - assert.ok(producer, 'the producer phase is republished'); + const producer = result.fixture_preparation; + assert.ok(producer, 'the producer phase is retained separately from consumer grading'); assert.ok( producer.steps.some(step => step.passed === false), 'with its failing step intact' ); + assert.equal( + result.phases.flatMap(phase => phase.steps).filter(step => !step.passed && !step.skipped).length, + result.failed_count + ); + const { formatStoryboardResultsAsJUnit } = require('../../dist/lib/testing/storyboard/junit.js'); + assert.match(formatStoryboardResultsAsJUnit([result]), /failures="1"/); + assert.deepEqual( + producer.steps.map(step => step.error), + state.phase.steps.map(step => step.error) + ); }); test(`${mode}: a standalone reuse carries the same producer records`, async () => { @@ -2148,6 +2182,7 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { }); assert.equal(result.passed, false); assert.deepEqual(result.fixture_resolutions, state.resolutions); + assert.ok(result.fixture_preparation); const check = result.validations.find(v => v.check === 'fixture_capture_unbuildable'); assert.ok(check, JSON.stringify(result.validations)); assert.equal(check.actual, 'preparation_failed'); @@ -2181,12 +2216,11 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { assert.ok(failing.request, `${mode}: the dispatched request is recorded`); assert.equal(failing.request.operation, 'get_products', mode); assert.equal(failing.request.payload.buying_mode, 'wholesale', mode); - // A connection-level failure is converted by the task executor into - // an error RESULT rather than a throw, so every one of these four - // arms does come back with something. What is recorded is that real - // payload — the runner never manufactures one. The `response: null` - // arm exists for an executor that genuinely throws. - assert.ok(failing.response_record, `${mode}: the raw response is recorded`); + if (mode === 'transport') { + assert.equal(failing.response_record, undefined, 'an SDK connection error is not a seller response'); + continue; + } + assert.ok(failing.response_record, `${mode}: an actual response is recorded`); assert.equal(failing.response_record.transport, 'mcp', mode); } }); @@ -2198,12 +2232,252 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { assert.ok(check, JSON.stringify(failing.validations)); assert.equal(check.passed, false); assert.equal(check.actual, 'schema_validation_failed'); - assert.match(check.schema_id, /get-products-response\.json$/); - assert.match(check.schema_url, /^https:\/\/adcontextprotocol\.org\/schemas\//); + assert.equal(check.schema_id, '/schemas/3.2.5/media-buy/get-products-response.json'); + assert.equal(check.schema_url, 'https://adcontextprotocol.org/schemas/3.2.5/media-buy/get-products-response.json'); assert.ok(check.request, 'the schema failure cites the request it graded'); assert.ok(check.response, 'and the response it graded'); }); + test('a real executor throw records a request without inventing a response', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => { + throw new Error('executor failed before any response'); + }; + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { ...transport([SELECTED]).options, agentTools: ['get_products'] }, + {} + ); + const failing = result.phase.steps.find(step => !step.passed); + assert.equal(failing.request.operation, 'get_products'); + assert.equal(failing.response_record, undefined); + }); + + test('A2A discovery evidence records the configured transport', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => ({ + success: true, + status: 'completed', + data: { + status: 'completed', + products: [{ product_id: 'malformed' }], + }, + }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { ...transport([SELECTED]).options, protocol: 'a2a', agentTools: ['get_products'] }, + {} + ); + const failing = result.phase.steps.find(step => !step.passed); + assert.equal(failing.request.transport, 'a2a'); + assert.equal(failing.response_record.transport, 'a2a'); + }); + + test('protocol and pagination failures are not relabelled as schema failures', async () => { + for (const mode of ['protocol', 'pagination']) { + const { state } = await prepareFailing(mode); + assert.equal( + state.phase.steps.flatMap(step => step.validations).some(v => v.check === 'response_schema'), + false, + mode + ); + } + }); + + test('the terminal failing page is retained even beyond the successful-page evidence budget', async () => { + let pages = 0; + const io = transport([], { + read: () => { + pages++; + return pages <= 16 + ? { + status: 'completed', + products: [], + cache_scope: 'account', + wholesale_feed_version: 'v1', + pagination: { has_more: true, cursor: `page-${pages + 1}` }, + } + : { status: 'completed', products: [{ product_id: 'late-malformed' }] }; + }, + }); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed'); + assert.equal(pages, 17); + const failing = state.phase.steps.find(step => !step.passed); + assert.equal(failing.request.payload.pagination.cursor, 'page-17'); + assert.match(JSON.stringify(failing.response_record.payload), /late-malformed/); + assert.doesNotMatch(JSON.stringify(failing.response_record.payload), /response_not_retained/); + }); + + test('captured protocol text redacts secret fields before producer evidence is retained', async () => { + const io = transport([], { + protocolFailure: true, + protocolPayload: { + status: 'failed', + adcp_error: { + code: 'INVALID_REQUEST', + message: 'Denied', + recovery: 'terminal', + details: { token: 'wire-secret-value', private_key: 'private-key-value', signature: 'signature-value' }, + }, + }, + }); + const state = await prepareStoryboardFixtures(ENDPOINT, board(), io.options); + assert.equal(state.disposition.status, 'failed'); + assert.doesNotMatch(JSON.stringify(state.phase), /wire-secret-value|private-key-value|signature-value/); + assert.match(JSON.stringify(state.phase), /redacted/); + }); + + test('A2A failed tasks retain the seller rejection without grading a product schema', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const response = { + jsonrpc: '2.0', + id: 'a2a-error', + result: { + kind: 'task', + id: 'failed-discovery', + contextId: 'ctx', + status: { state: 'failed' }, + artifacts: [ + { + artifactId: 'error', + parts: [ + { + kind: 'data', + data: { + adcp_error: { code: 'INVALID_REQUEST', message: 'Seller refused discovery', recovery: 'terminal' }, + }, + }, + ], + }, + ], + }, + }; + const dispatch = async () => ({ + success: false, + error: 'Seller refused discovery', + data: {}, + debug_logs: [{ message: 'A2A: Response received', response }], + }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { + ...transport([]).options, + protocol: 'a2a', + agentTools: ['get_products'], + }, + {} + ); + const failed = result.phase.steps.find(step => !step.passed); + assert.equal(failed.task, 'get_products'); + assert.deepEqual(failed.response_record.payload, response); + assert.equal( + failed.validations.some(v => v.check === 'response_schema'), + false + ); + }); + + test('schema diagnostics use the discovery route version and the terminal SDK trace', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const malformed = { status: 'completed', products: [{ product_id: 'terminal-malformed' }] }; + const protocolResponse = { content: [{ type: 'text', text: JSON.stringify(malformed) }] }; + const dispatch = async () => ({ + success: false, + error: 'validation failed', + data: { products: [] }, + debug_logs: [ + { + message: 'MCP: Tool get_products response received', + response: { content: [{ type: 'text', text: '{"status":"submitted"}' }] }, + }, + { message: 'MCP: Tool get_products response received', response: protocolResponse }, + ], + }); + const client = { getProducts: dispatch, getProductsLegacy: dispatch }; + const options = { ...transport([]).options, adcpVersion: '3.2.6', agentTools: ['get_products'] }; + const result = await runControllerSeeding(client, board(), options, {}, client, () => ({ + client, + options: { ...options, adcpVersion: '3.2.5' }, + })); + const failed = result.phase.steps.find(step => !step.passed); + const check = failed.validations.find(v => v.check === 'response_schema'); + assert.equal(check.schema_id, '/schemas/3.2.5/media-buy/get-products-response.json'); + assert.equal(check.expected, check.schema_id); + assert.deepEqual(failed.response_record.payload, protocolResponse); + }); + + test('SDK-projected errors without wire evidence are never graded as seller schema failures', async () => { + const { runControllerSeeding } = require('../../dist/lib/testing/storyboard/seeding.js'); + const dispatch = async () => ({ success: false, error: 'connection failed', data: { products: [] } }); + const result = await runControllerSeeding( + { getProducts: dispatch, getProductsLegacy: dispatch }, + board(), + { + ...transport([]).options, + agentTools: ['get_products'], + }, + {} + ); + const failed = result.phase.steps.find(step => !step.passed); + assert.equal(failed.response_record, undefined); + assert.equal( + failed.validations.some(v => v.check === 'response_schema'), + false + ); + }); + + test('a partial graph keeps protected captures out of consumer context without duplicating failures', async () => { + const doc = board(); + doc.fixtures.products.push(product('other-product')); + doc.fixture_resolution.products.push({ handle: 'other-product', strategies: ['seed'] }); + const io = transport([SELECTED], { controller: () => ({ success: false, error: 'UPSTREAM_FAILURE' }) }); + const state = await prepareStoryboardFixtures(ENDPOINT, doc, { + ...io.options, + agentTools: [...TOOLS, 'comply_test_controller'], + }); + assert.equal(state.disposition.status, 'failed', JSON.stringify(state.disposition)); + assert.ok( + state.phase.steps.some(step => step.passed), + 'the graph must contain earlier successful producer work' + ); + const runIo = transport([SELECTED]); + const result = await runStoryboard(ENDPOINT, doc, { ...runIo.options, ...withPreparedFixtures(state, ALLOW) }); + assert.equal(result.failed_count, 1); + assert.equal(result.phases.flatMap(phase => phase.steps).length, 1); + assert.doesNotMatch(JSON.stringify(result.fixture_preparation.steps.map(step => step.context)), /__adcp_fixture_/); + assert.deepEqual( + result.fixture_preparation.steps.map(step => step.error), + state.phase.steps.map(step => step.error) + ); + }); + + test('standalone denial retains failed producer evidence only in the private sink', async () => { + const { state } = await prepareFailing('schema'); + const io = transport([SELECTED]); + let audited; + await assert.rejects( + runStoryboardStep(ENDPOINT, board(), 'create', { + ...io.options, + prepared_fixtures: state, + prepared_fixtures_authorizer: () => ({ authorized: false, detail: 'denied' }), + prepared_fixtures_audit_sink: record => { + audited = record; + }, + }), + error => { + assert.ok(error instanceof PreparedFixturesDeniedError); + assert.doesNotMatch(JSON.stringify(error), /malformed|authored-product/); + return error.refusalCode === 'authorizer_denied'; + } + ); + assert.deepEqual(audited.fixture_resolutions, state.resolutions); + assert.deepEqual(audited.fixture_preparation, state.phase); + assert.deepEqual(io.wire, []); + }); + test('a denied caller still gets only the typed denial, with the record to the sink', async () => { const { state } = await prepareFailing('schema'); const io = transport([SELECTED]); @@ -2223,6 +2497,8 @@ describe('failed preparation keeps its producer evidence (#3385)', () => { // A failed preparation does not downgrade the denial: the caller gets // no producer records, and the host's copy is complete. assert.ok(audited, 'the sink received the record'); + assert.deepEqual(audited.fixture_resolutions, state.resolutions); + assert.deepEqual(audited.fixture_preparation, state.phase); assert.deepEqual(io.wire, []); }); });