From a1f308f0bc471352afc84d824b88435194fd473a Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 17 Oct 2026 13:00:53 +0000 Subject: [PATCH 1/2] fix(testing): validate capabilities against the echoed release under an external pin root Hosted compliance always runs with an external schema root for the pinned release, and 15.9.2's served-release rule let that root win over the echo. A 3.0 target therefore graded a conformant 3.2 get_adcp_capabilities response against 3.0: every storyboard response_schema check on that task failed (52 steps in the 3.0.27 suite), and discovery raised spurious capabilities_response_schema_invalid notices. For get_adcp_capabilities only, the storyboard response_schema check, discoverAgentProfile and TaskExecutor now validate against the echoed adcp_version when it is the pin's major, is listed in the response's own adcp.supported_versions, and has a schema root (one supplied for that release, an external root registered for it, or the SDK's bundled schemas), whether or not an external root is active for the pin. Otherwise they validate strictly against the pin. Other tasks, storyboard selection, the run's adcp_version and grading stay on the pin. New servedReleaseSchemaRoots option on comply(), runStoryboard() and runStoryboardStep(), and a withServedReleaseSchemaRoots() scope helper exported from @adcp/sdk/testing, let a hosted grader supply its own roots for echoed releases. They are activated only for that capabilities validation. Fixes #3320 Co-Authored-By: Claude Opus 5.5 (1M context) --- ...pabilities-echoed-release-external-root.md | 9 + docs/guides/VALIDATE-YOUR-AGENT.md | 34 ++ src/lib/core/TaskExecutor.ts | 126 +++--- src/lib/testing/client.ts | 15 +- src/lib/testing/compliance/comply.ts | 12 +- src/lib/testing/index.ts | 7 +- src/lib/testing/storyboard/runner.ts | 17 +- src/lib/testing/storyboard/validations.ts | 32 +- src/lib/testing/types.ts | 36 ++ src/lib/validation/index.ts | 1 + src/lib/validation/schema-loader.ts | 229 ++++++++-- ...s-echoed-release-hosted-validation.test.js | 410 ++++++++++++++++++ ...bilities-served-release-validation.test.js | 68 +-- 13 files changed, 857 insertions(+), 139 deletions(-) create mode 100644 .changeset/capabilities-echoed-release-external-root.md create mode 100644 test/lib/capabilities-echoed-release-hosted-validation.test.js diff --git a/.changeset/capabilities-echoed-release-external-root.md b/.changeset/capabilities-echoed-release-external-root.md new file mode 100644 index 000000000..040bf66cd --- /dev/null +++ b/.changeset/capabilities-echoed-release-external-root.md @@ -0,0 +1,9 @@ +--- +'@adcp/sdk': patch +--- + +Validate `get_adcp_capabilities` against the release the agent echoed even when an external schema root is active for the pin (#3320). Hosted compliance always runs with one, so a 3.0 target graded a conformant 3.2 capabilities response against 3.0: every storyboard `response_schema` check on that task failed, and discovery raised spurious `capabilities_response_schema_invalid` notices. + +The storyboard `response_schema` check, discovery's capability validation and `getAdcpCapabilities()` now use the echoed `adcp_version` when it is the pin's major, is listed in the response's own `adcp.supported_versions`, and has a schema root: one supplied for that release, an external root registered for it, or the SDK's bundled schemas. Otherwise they validate strictly against the pin. Requiring the `supported_versions` listing is new. Other tasks, storyboard selection, the run's `adcp_version` and grading stay on the pin. + +New `servedReleaseSchemaRoots` option on `comply()`, `runStoryboard()` and `runStoryboardStep()`: a map of AdCP version to schema root (same shape as `schemaRoot`) for releases an agent may echo. Those roots are used only for this capabilities validation. `withServedReleaseSchemaRoots(roots, fn)` from `@adcp/sdk/testing` scopes the same map for other callers. diff --git a/docs/guides/VALIDATE-YOUR-AGENT.md b/docs/guides/VALIDATE-YOUR-AGENT.md index a947ec48c..dde300d4f 100644 --- a/docs/guides/VALIDATE-YOUR-AGENT.md +++ b/docs/guides/VALIDATE-YOUR-AGENT.md @@ -187,6 +187,40 @@ await comply(agentUrl, { }); ``` +**Capabilities served at a newer release.** A 3.0 pin sends only +`adcp_major_version: 3`, so a conformant agent may serve and echo its highest +3.x release in `get_adcp_capabilities` (versioning spec, "Server echoes"). The +runner validates that one response against the echoed release when the echo is +the pin's major, is listed in the response's own `adcp.supported_versions`, and +has a schema root. That covers discovery's `capabilities_response_schema_invalid` +notices, `getAdcpCapabilities()` and every storyboard `response_schema` check on +`get_adcp_capabilities`, even with `schemaRoot` set for the pin. Otherwise it +validates strictly against the pin. Every other task's response, storyboard +selection, the run's `adcp_version` and grading stay on the pin. + +The echoed release's schemas come from the SDK's bundle by default. A grader +that ships its own roots per release can supply them with +`servedReleaseSchemaRoots` (on `comply()`, `runStoryboard()` and +`runStoryboardStep()`), keyed by AdCP version: + +```ts +await comply(agentUrl, { + version: '3.0.27', + complianceDir: '/app/dist/compliance/3.0.27', + schemaRoot: '/app/dist/schemas/3.0.27', + servedReleaseSchemaRoots: { + '3.1.29': '/app/dist/schemas/3.1.29', + '3.2.5': '/app/dist/schemas/3.2.5', + }, +}); +``` + +Those roots are used only for that capabilities validation. Each one is checked +against its key once per process, and a mismatched or missing root throws +before the run starts. Callers that validate outside `comply()` and the runner +can scope the same map with `withServedReleaseSchemaRoots(roots, fn)` from +`@adcp/sdk/testing`. + **Multi-agent storyboards under `comply()`.** `comply()` grades one agent, so storyboards that declare `requires: [multi_agent]` (for example the governance-aware seller scenarios) skip with `requirement_unmet` and cap their diff --git a/src/lib/core/TaskExecutor.ts b/src/lib/core/TaskExecutor.ts index a7fc4fcd7..fb73b18f0 100644 --- a/src/lib/core/TaskExecutor.ts +++ b/src/lib/core/TaskExecutor.ts @@ -34,7 +34,7 @@ import { type ValidationMode, } from '../validation/client-hooks'; import { formatIssues } from '../validation/schema-validator'; -import { resolveServedValidationVersion } from '../validation/schema-loader'; +import { withServedValidationVersion } from '../validation/schema-loader'; import { ADCP_VERSION } from '../version'; import { unwrapProtocolResponse, isAdcpOperationSuccess } from '../utils/response-unwrapper'; import { extractAdcpErrorInfo, extractCorrelationId } from '../utils/error-extraction'; @@ -6125,64 +6125,31 @@ export class TaskExecutor { // Normal dispatch uses lastKnownServerVersion; restart recovery passes // the persisted continuation version explicitly so concurrent or fresh // executors cannot validate the seller's response against another wire. - let validationVersion = + const pinnedValidationVersion = (serverVersion ?? this.lastKnownServerVersion) === 'v2' ? 'v2.5' : (this.responseAdcpVersionForValidation() ?? this.config.adcpVersion ?? ADCP_VERSION); // Capabilities are negotiated before any pin applies: a 3.0 pin (or a // major-only envelope) is served the seller's highest release in the // major, echoed in `adcp_version`. Validate against that release, as the - // versioning spec requires ("Server echoes"), falling back to the pin - // when there is no usable echo. Only the validator's version changes; - // the unwrapper keeps `effectiveResponseAdcpVersion()`. - if (taskName === 'get_adcp_capabilities' && validationVersion !== 'v2.5') { - validationVersion = resolveServedValidationVersion(response, validationVersion); - } - const normalizedResponse = this.normalizeResponseForValidation(response, taskName, validationVersion === 'v2.5'); - // TaskExecutor owns the user-facing log entry below. Passing debugLogs - // into the lower-level hook would emit a second, unversioned duplicate. - const outcome = - (this.config.experimentalIdentity33?.() || - (this.config.identity33SchemaExperiment && - normalizedResponse && - typeof normalizedResponse === 'object' && - isAdcpVersionAtLeast((normalizedResponse as { adcp_version?: string }).adcp_version, '3.3'))) && - mode !== 'off' - ? validateIdentity33Payload(taskName, 'response', normalizedResponse, validationVersion) - : validateIncomingResponse(taskName, normalizedResponse, mode, undefined, validationVersion); - if (outcome.valid) return { valid: true, errors: [] }; - - const errorStrings = outcome.issues.map(i => `${i.pointer}: ${i.message}`); - - if (logViolations) { - debugLogs.push({ - timestamp: new Date().toISOString(), - type: mode === 'strict' ? 'error' : 'warning', - message: `Schema validation ${mode === 'strict' ? 'failed' : 'warning'} for ${taskName}: ${formatIssues( - outcome.issues - )}`, - errors: errorStrings, - issues: outcome.issues, - schemaVariant: outcome.variant, - schemaVersion: validationVersion, - mode, - }); - } - - if (mode === 'warn') { - if (logViolations) { - console.warn( - `Schema validation failed for ${taskName} against AdCP ${validationVersion} (non-blocking):`, - errorStrings - ); - } - return { valid: true, errors: [] }; - } - - if (logViolations) { - console.error(`Schema validation failed for ${taskName} against AdCP ${validationVersion}:`, errorStrings); + // versioning spec requires ("Server echoes"), when it is same-major, + // listed in the response's `supported_versions` and has a schema root; + // otherwise against the pin. This holds even when an external schema + // root is active for the pin. Only the validator's version changes; the + // unwrapper keeps `effectiveResponseAdcpVersion()`. + if (taskName === 'get_adcp_capabilities' && pinnedValidationVersion !== 'v2.5') { + return withServedValidationVersion(response, pinnedValidationVersion, validationVersion => + this.validateResponseSchemaAgainst(response, taskName, debugLogs, mode, logViolations, validationVersion) + ); } - return { valid: false, errors: errorStrings }; + return this.validateResponseSchemaAgainst( + response, + taskName, + debugLogs, + mode, + logViolations, + pinnedValidationVersion + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'unknown error'; console.error('Error during schema validation:', errorMessage); @@ -6193,6 +6160,61 @@ export class TaskExecutor { } } + private validateResponseSchemaAgainst( + response: any, + taskName: string, + debugLogs: any[], + mode: ValidationMode, + logViolations: boolean, + validationVersion: string + ): { valid: boolean; errors: string[] } { + const normalizedResponse = this.normalizeResponseForValidation(response, taskName, validationVersion === 'v2.5'); + // TaskExecutor owns the user-facing log entry below. Passing debugLogs + // into the lower-level hook would emit a second, unversioned duplicate. + const outcome = + (this.config.experimentalIdentity33?.() || + (this.config.identity33SchemaExperiment && + normalizedResponse && + typeof normalizedResponse === 'object' && + isAdcpVersionAtLeast((normalizedResponse as { adcp_version?: string }).adcp_version, '3.3'))) && + mode !== 'off' + ? validateIdentity33Payload(taskName, 'response', normalizedResponse, validationVersion) + : validateIncomingResponse(taskName, normalizedResponse, mode, undefined, validationVersion); + if (outcome.valid) return { valid: true, errors: [] }; + + const errorStrings = outcome.issues.map(i => `${i.pointer}: ${i.message}`); + + if (logViolations) { + debugLogs.push({ + timestamp: new Date().toISOString(), + type: mode === 'strict' ? 'error' : 'warning', + message: `Schema validation ${mode === 'strict' ? 'failed' : 'warning'} for ${taskName}: ${formatIssues( + outcome.issues + )}`, + errors: errorStrings, + issues: outcome.issues, + schemaVariant: outcome.variant, + schemaVersion: validationVersion, + mode, + }); + } + + if (mode === 'warn') { + if (logViolations) { + console.warn( + `Schema validation failed for ${taskName} against AdCP ${validationVersion} (non-blocking):`, + errorStrings + ); + } + return { valid: true, errors: [] }; + } + + if (logViolations) { + console.error(`Schema validation failed for ${taskName} against AdCP ${validationVersion}:`, errorStrings); + } + return { valid: false, errors: errorStrings }; + } + private responseAdcpVersionForValidation(): string | undefined { if (this.config.versionEnvelope === 'major-only') return '3.0'; return undefined; diff --git a/src/lib/testing/client.ts b/src/lib/testing/client.ts index bf9b411b0..32918a463 100644 --- a/src/lib/testing/client.ts +++ b/src/lib/testing/client.ts @@ -28,7 +28,7 @@ import type { AgentRequestSigningConfig } from '../types/adcp'; import { createHmac, hkdfSync, randomBytes } from 'node:crypto'; import { AsyncLocalStorage } from 'node:async_hooks'; import { validateIncomingResponse } from '../validation/client-hooks'; -import { resolveServedValidationVersion } from '../validation/schema-loader'; +import { withServedValidationVersion } from '../validation/schema-loader'; import { extractVersionUnsupportedDetails } from '../utils/error-extraction'; import { buildAgentSigningContext } from '../signing/client'; import type { VerifierCapability } from '../signing/types'; @@ -848,13 +848,14 @@ export async function discoverAgentProfile( // Validate against the release the seller served (its echoed // `adcp_version`), not the pin. A 3.0 pin negotiates major-only, so // the seller serves its highest 3.x release (versioning spec, - // "Server echoes"). Falls back to the pin when there is no usable echo. - const validation = validateIncomingResponse( - 'get_adcp_capabilities', + // "Server echoes"). The echo is used when it is same-major, listed in + // the response's `supported_versions` and has a schema root, even + // with an external root active for the pin; otherwise the pin. + const validation = withServedValidationVersion( caps.data, - 'strict', - undefined, - resolveServedValidationVersion(caps.data, schemaAdcpVersion ?? client.getAdcpVersion()) + schemaAdcpVersion ?? client.getAdcpVersion(), + validationVersion => + validateIncomingResponse('get_adcp_capabilities', caps.data, 'strict', undefined, validationVersion) ); if (!validation.valid) { profile.capabilities_schema_issues = validation.issues.map(issue => ({ diff --git a/src/lib/testing/compliance/comply.ts b/src/lib/testing/compliance/comply.ts index 166f3efe7..4b455fa7a 100644 --- a/src/lib/testing/compliance/comply.ts +++ b/src/lib/testing/compliance/comply.ts @@ -80,7 +80,7 @@ import { detectController, hasTestController } from '../test-controller'; import type { ControllerDetection } from '../test-controller'; import { randomBytes } from 'crypto'; import { isPre31AdcpVersion } from '../../utils/adcp-version-config'; -import { withExternalSchemaRoot } from '../../validation/schema-loader'; +import { withExternalSchemaRoot, withServedReleaseSchemaRoots } from '../../validation/schema-loader'; import { redactOAuthUrlForOutput, redactOAuthUrlsInText } from '../storyboard/oauth-metadata-graph'; import { LIBRARY_VERSION } from '../../version'; import { validationFailsStep } from '../storyboard/validations'; @@ -2287,7 +2287,13 @@ async function complyImpl(agentUrl: string, options: ComplyOptions): Promise { + // `servedReleaseSchemaRoots` stays in `testOptions` so routed and nested + // storyboard runs see it too; scoping it here also covers discovery. + return await withExternalSchemaRoot(complianceIndex.adcp_version, scopedSchemaRoot, () => + withServedReleaseSchemaRoots(testOptions.servedReleaseSchemaRoots, () => complyInSchemaScope()) + ); + + async function complyInSchemaScope(): Promise { let effectiveOptions: TestOptions = applyAdcpVersionRunOptions(complianceIndex.adcp_version, { ...testOptions, // `comply()` has always defaulted an unset transport to MCP when it built @@ -2725,7 +2731,7 @@ async function complyImpl(agentUrl: string, options: ComplyOptions): Promise + const runBody = () => + withServedReleaseSchemaRoots(options.servedReleaseSchemaRoots, () => runStoryboardBody(agentUrlOrUrls, storyboard, options) ); + if (schemaRoot) { + return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, runBody); } - return await runStoryboardBody(agentUrlOrUrls, storyboard, options); + return await runBody(); }, { isolate: !hasTestClientScope() } ); @@ -5489,12 +5492,14 @@ export async function runStoryboardStep( options = resolveDeclaredTestKit(storyboard, options); options = applyNativeA2AComplianceTransportOptions(options); const schemaRoot = getRunSchemaRoot(options); - if (schemaRoot) { - return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, () => + const runBody = () => + withServedReleaseSchemaRoots(options.servedReleaseSchemaRoots, () => runStoryboardStepBody(agentUrl, storyboard, stepId, options) ); + if (schemaRoot) { + return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, runBody); } - return await runStoryboardStepBody(agentUrl, storyboard, stepId, options); + return await runBody(); }, { isolate: !hasTestClientScope() } ); diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index 4e0c2a5e4..79b8f7bb4 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -12,7 +12,11 @@ import { prepareResponseForSchemaValidation, TOOL_RESPONSE_SCHEMAS } from '../.. import { injectLegacyEnvelopeStatus } from '../../utils/envelope-status-compat'; import { TRANSPORT_SUFFIX_REGEX } from '../../utils/a2a-discovery'; import { validateResponse, type ValidationIssue } from '../../validation/schema-validator'; -import { hasSchemaBundle, isExternalSchemaRootActive } from '../../validation/schema-loader'; +import { + hasSchemaBundle, + isExternalSchemaRootActive, + withServedValidationVersion, +} from '../../validation/schema-loader'; import { ADCP_VERSION, LIBRARY_VERSION } from '../../version'; import { isPre31AdcpVersion } from '../../utils/adcp-version-config'; import { gte as semverGte, valid as validSemver } from 'semver'; @@ -536,6 +540,32 @@ function validateResponseSchema( validation: StoryboardValidation, ctx: ValidationContext, taskResult: TaskResult +): ValidationResult { + // Capabilities are negotiated before any pin applies: a 3.0 pin is served + // the agent's highest release in the major, echoed in `adcp_version`. + // Validate against that release (versioning spec, "Server echoes") when it + // is same-major, listed in the response's `supported_versions` and has a + // schema root, even when an external root is active for the pin. Otherwise + // keep the pin. Every other task stays on the pin. + const pinnedVersion = ctx.adcpVersion; + if (ctx.taskName === 'get_adcp_capabilities' && pinnedVersion !== undefined) { + return withServedValidationVersion(taskResult.data, pinnedVersion, validationVersion => + validateResponseSchemaForVersion( + validation, + validationVersion === pinnedVersion + ? ctx + : { ...ctx, adcpVersion: validationVersion, responseAdcpVersion: validationVersion }, + taskResult + ) + ); + } + return validateResponseSchemaForVersion(validation, ctx, taskResult); +} + +function validateResponseSchemaForVersion( + validation: StoryboardValidation, + ctx: ValidationContext, + taskResult: TaskResult ): ValidationResult { const taskName = ctx.taskName; const schemaIdentityVersion = ctx.adcpVersion ?? ctx.responseAdcpVersion ?? ADCP_VERSION; diff --git a/src/lib/testing/types.ts b/src/lib/testing/types.ts index 0d6da32d1..5679feec4 100644 --- a/src/lib/testing/types.ts +++ b/src/lib/testing/types.ts @@ -126,6 +126,42 @@ export interface TestOptions { * authoritative over the SDK's generated Zod snapshot for the run. */ schemaRoot?: string; + /** + * Schema roots for releases the agent may echo in its + * `get_adcp_capabilities` response, keyed by AdCP version (`'3.2'`, + * `'3.2.5'`, `'3.3.0-beta.1'`). Values are schema-data roots in the same + * shape as `schemaRoot`. + * + * A 3.0 pin sends only `adcp_major_version`, so an agent may serve and echo + * a newer release in the same major (versioning spec, "Server echoes"). + * The SDK validates that capabilities response against the echoed release + * when the echo is the pin's major, is listed in the response's + * `adcp.supported_versions`, and has a schema root: an entry here, an + * external root registered for it, or the SDK's bundled schemas. That + * applies to discovery's capability notices, `getAdcpCapabilities()` and + * storyboard `response_schema` checks on `get_adcp_capabilities`, including + * when `schemaRoot` is set for the pin. Otherwise validation stays on the + * pin. + * + * Entries are used for that capabilities validation only. Every other + * task's response, storyboard selection and grading stay on the pin. Each + * root is checked against its key once per process; a malformed map throws + * before the run starts. Honored by `comply()`, `runStoryboard()` and + * `runStoryboardStep()`; other callers can use `withServedReleaseSchemaRoots`. + * + * @example + * ```ts + * await comply(agentUrl, { + * complianceDir: hosted30.complianceDir, + * schemaRoot: hosted30.schemaRoot, + * servedReleaseSchemaRoots: { + * '3.1.29': hosted31.schemaRoot, + * '3.2.5': hosted32.schemaRoot, + * }, + * }); + * ``` + */ + servedReleaseSchemaRoots?: Readonly>; /** * Whether strict AJV response-schema failures contribute to storyboard * grading. Defaults to `true`, matching the hosted compliance grader. diff --git a/src/lib/validation/index.ts b/src/lib/validation/index.ts index 9f15a7ff2..989f4c859 100644 --- a/src/lib/validation/index.ts +++ b/src/lib/validation/index.ts @@ -17,6 +17,7 @@ export { registerExternalSchemaRoot, unregisterExternalSchemaRoot, withExternalSchemaRoot, + withServedReleaseSchemaRoots, } from './schema-loader'; export type { Direction, ResponseVariant } from './schema-loader'; export { validateOutgoingRequest, validateIncomingResponse, resolveValidationModes } from './client-hooks'; diff --git a/src/lib/validation/schema-loader.ts b/src/lib/validation/schema-loader.ts index d0e76b89a..4ada3c43d 100644 --- a/src/lib/validation/schema-loader.ts +++ b/src/lib/validation/schema-loader.ts @@ -1280,70 +1280,221 @@ export function hasSchemaBundle(version: string): boolean { } } +/** + * A caller-supplied schema root for a release an agent may echo in a + * `get_adcp_capabilities` response. `aliases` is the validated alias set + * (computed once per process per `version`/`root` pair). + */ +interface ServedReleaseSchemaRoot { + readonly version: string; + readonly root: string; + readonly aliases: ReadonlySet; +} + +const servedReleaseSchemaRootScope = new AsyncLocalStorage(); +/** + * Validated alias sets for served-release roots, keyed by `version\0root`. + * Checking a root walks every schema file in it (about a second for a full + * release), so a hosted caller that passes the same roots on every run pays + * that once per process. Roots are treated as immutable artifacts. + */ +const servedReleaseRootAliases = new Map>(); + +/** + * Run `fn` with schema roots for the releases an agent may echo in its + * `get_adcp_capabilities` response. + * + * Keys are AdCP versions in any form the SDK accepts for a pin (`'3.2'`, + * `'3.2.5'`, `'3.3.0-beta.1'`); values are schema-data roots in the same + * shape as `schemaRoot` (a directory holding `bundled/`, e.g. a hosted + * `dist/schemas/3.2.5`). Each root is checked against its key the same way + * `withExternalSchemaRoot` checks one, and a malformed map throws before `fn` + * runs. + * + * The roots are consulted only when validating a `get_adcp_capabilities` + * response against the release the agent echoed (see + * {@link resolveServedValidationVersion}): discovery's capability notices, + * `getAdcpCapabilities()` response validation and storyboard + * `response_schema` checks on that task. They are not activated for any other + * task, so every other response, storyboard selection and grading stay on the + * pin. A supplied root takes precedence over an external root registered for + * the same release and over the SDK's bundled schemas for it. + * + * Scoped to the current async execution chain; nested calls add to the + * enclosing scope, with the innermost entry for a release winning. + * `comply()` and `runStoryboard()` accept the same map as + * `servedReleaseSchemaRoots` and call this for you. + */ +export function withServedReleaseSchemaRoots(roots: Readonly> | undefined, fn: () => T): T { + if (roots === undefined) return fn(); + if (roots === null || typeof roots !== 'object' || Array.isArray(roots)) { + throw new TypeError('servedReleaseSchemaRoots must be an object mapping AdCP versions to schema roots'); + } + const entries: ServedReleaseSchemaRoot[] = []; + for (const [version, root] of Object.entries(roots)) { + resolveBundleKey(version); + if (typeof root !== 'string' || root.length === 0) { + throw new TypeError(`servedReleaseSchemaRoots["${version}"] must be a non-empty schema root path`); + } + const memoKey = `${version}\0${root}`; + let aliases = servedReleaseRootAliases.get(memoKey); + if (!aliases) { + if (!hasSchemaRootShape(root)) { + throw new Error(`Served-release AdCP schema root for version "${version}" not found or empty at ${root}`); + } + aliases = assertSchemaRootMatchesVersion(version, root); + servedReleaseRootAliases.set(memoKey, aliases); + } + entries.push({ version, root, aliases }); + } + if (entries.length === 0) return fn(); + const parent = servedReleaseSchemaRootScope.getStore() ?? []; + return servedReleaseSchemaRootScope.run([...entries, ...parent], fn); +} + +function findServedReleaseSchemaRoot(release: string): ServedReleaseSchemaRoot | undefined { + const entries = servedReleaseSchemaRootScope.getStore(); + if (!entries?.length) return undefined; + let key: string; + try { + key = resolveBundleKey(release); + } catch { + return undefined; + } + return entries.find(entry => entry.aliases.has(key) || entry.aliases.has(release)); +} + /** Major of an echoed release-precision `adcp_version` (`core/version-envelope.json` wire pattern). */ const SERVED_RELEASE_MAJOR = /^(\d+)\.\d+(?:-[a-zA-Z0-9.-]+)?$/; /** Major of a numeric pin: `'3.0'`, `'3.0.25'`, `'3.1-beta'`, `'3.1.0-beta.1'`. */ const NUMERIC_PIN_MAJOR = /^(\d+)\.\d+/; +/** Whether the response's own `adcp.supported_versions` lists `release`. */ +function listsSupportedRelease(response: object, release: string): boolean { + const adcp = (response as { adcp?: unknown }).adcp; + if (adcp === null || typeof adcp !== 'object') return false; + const supported = (adcp as { supported_versions?: unknown }).supported_versions; + if (!Array.isArray(supported)) return false; + return supported.some(entry => { + if (typeof entry !== 'string') return false; + if (entry === release) return true; + try { + return toReleasePrecisionWire(entry) === release; + } catch { + return false; + } + }); +} + +interface ServedValidationSelection { + /** Version to validate against: the echoed release, or the pin. */ + version: string; + /** Caller-supplied root for the echoed release, when one applies. */ + served?: ServedReleaseSchemaRoot; +} + +function selectServedValidationVersion(response: unknown, pinnedVersion: string): ServedValidationSelection { + const pin: ServedValidationSelection = { version: pinnedVersion }; + const pinMajor = NUMERIC_PIN_MAJOR.exec(pinnedVersion)?.[1]; + // Legacy aliases ('v2.5', 'v3') have their own adapter paths; leave them alone. + if (pinMajor === undefined) return pin; + if (response === null || typeof response !== 'object') return pin; + + const echoed = (response as { adcp_version?: unknown }).adcp_version; + if (typeof echoed !== 'string') return pin; + const echoMajor = SERVED_RELEASE_MAJOR.exec(echoed)?.[1]; + if (echoMajor === undefined || echoMajor !== pinMajor) return pin; + + let pinnedRelease: string; + try { + pinnedRelease = toReleasePrecisionWire(pinnedVersion); + } catch { + return pin; + } + // The pinned release itself: keep the exact pin so a patch-precise pin and + // any external root bound to it still apply. + if (echoed === pinnedRelease) return pin; + // An agent may only serve a release it advertises. + if (!listsSupportedRelease(response, echoed)) return pin; + + const served = findServedReleaseSchemaRoot(echoed); + if (served) return { version: echoed, served }; + // Covers an external root registered (or scoped) for the echo and the + // SDK's bundled schemas. + return hasSchemaBundle(echoed) ? { version: echoed } : pin; +} + /** - * Select the AdCP release a seller response is validated against. + * Select the AdCP release a `get_adcp_capabilities` response is validated + * against. * * The versioning spec (`docs/reference/versioning.mdx`, "Server echoes") says * the seller echoes the release it actually served in `adcp_version`, and * buyers SHOULD validate against that release's schema, not their pin; with - * no echo, against the pin. This matters most for `get_adcp_capabilities` - * under a 3.0 pin: the 3.0 bundle has no `adcp_version` request field, so the - * SDK sends only `adcp_major_version: 3` and the seller serves (and echoes) - * its highest 3.x release. + * no echo, against the pin. This matters most under a 3.0 pin: the 3.0 + * bundle has no `adcp_version` request field, so the SDK sends only + * `adcp_major_version: 3` and the seller serves (and echoes) its highest 3.x + * release. * - * Returns the version to validate `response` against: the echoed - * `adcp_version` when it is a usable release, otherwise `pinnedVersion`. + * Returns the echoed `adcp_version` when all of these hold, otherwise + * `pinnedVersion`: + * - it is a release-precision string in the same major as the pin; + * - the response's own `adcp.supported_versions` lists it; + * - a schema root for it is available: one supplied through + * {@link withServedReleaseSchemaRoots} (`servedReleaseSchemaRoots` on + * `comply()` / `runStoryboard()`), an external root registered or scoped + * for it, or the SDK's bundled schemas. * - * The echo is used only when it is a release-precision string in the same - * major as the pin and the SDK has a schema bundle for it. Validation stays - * strict either way: an echo from another major, a malformed echo, or one the - * SDK cannot validate falls back to the pin rather than skipping validation. - * When the echo names the pinned release itself, the exact pin is returned so - * a patch-precise pin (and any external schema root bound to it) still - * applies. `supported_versions` is deliberately not consulted: with no echo - * the spec says to validate against the pin. + * This holds whether or not an external schema root is active for the pin. + * Hosted compliance always runs with one, and the capabilities response is + * negotiated before any pin applies, so the pin's root must not override the + * release the agent actually served. * - * Two deliberate consequences: - * - An external schema root the caller bound to the pin (`schemaRoot`, - * `withExternalSchemaRoot`) wins over the echo, matching the storyboard - * runner's `response_schema` precedence: the caller chose that protocol - * source explicitly. - * - An echo older than the pin (pin 3.2, served 3.0) is validated against the - * older release. That is the spec's downshift rule, not a loosening bug. + * Validation stays strict either way: an echo from another major, a malformed + * or unlisted echo, or one with no schema available falls back to the pin + * rather than skipping validation. When the echo names the pinned release + * itself, the exact pin is returned so a patch-precise pin (and any external + * schema root bound to it) still applies. An echo older than the pin (pin + * 3.2, served 3.0) is validated against the older release; that is the spec's + * downshift rule, not a loosening bug. + * + * Callers apply this to `get_adcp_capabilities` only. Use + * {@link withServedValidationVersion} to also activate a supplied root. * * @internal — not part of the public API surface; may change without a major bump. */ export function resolveServedValidationVersion(response: unknown, pinnedVersion: string): string { - const pinMajor = NUMERIC_PIN_MAJOR.exec(pinnedVersion)?.[1]; - // Legacy aliases ('v2.5', 'v3') have their own adapter paths; leave them alone. - if (pinMajor === undefined) return pinnedVersion; try { - if (isExternalSchemaRootActive(pinnedVersion)) return pinnedVersion; + return selectServedValidationVersion(response, pinnedVersion).version; } catch { return pinnedVersion; } +} - const echoed = - response !== null && typeof response === 'object' - ? (response as { adcp_version?: unknown }).adcp_version - : undefined; - if (typeof echoed !== 'string') return pinnedVersion; - const echoMajor = SERVED_RELEASE_MAJOR.exec(echoed)?.[1]; - if (echoMajor === undefined || echoMajor !== pinMajor) return pinnedVersion; - - let pinnedRelease: string; +/** + * Run `fn` with the release {@link resolveServedValidationVersion} selects. + * When that release has a root supplied through + * {@link withServedReleaseSchemaRoots}, the root is active for the duration + * of `fn` only, so it never leaks into validation of any other response. + * + * @internal — not part of the public API surface; may change without a major bump. + */ +export function withServedValidationVersion( + response: unknown, + pinnedVersion: string, + fn: (version: string) => T +): T { + let selection: ServedValidationSelection; try { - pinnedRelease = toReleasePrecisionWire(pinnedVersion); + selection = selectServedValidationVersion(response, pinnedVersion); } catch { - return pinnedVersion; + selection = { version: pinnedVersion }; } - if (echoed === pinnedRelease) return pinnedVersion; - return hasSchemaBundle(echoed) ? echoed : pinnedVersion; + const { version, served } = selection; + if (!served) return fn(version); + const next = new Map(scopedExternalSchemaRoots.getStore()); + for (const alias of served.aliases) next.set(alias, served.root); + return scopedExternalSchemaRoots.run(next, () => fn(version)); } const mcpToolSchemaCache = new Map> | null>(); diff --git a/test/lib/capabilities-echoed-release-hosted-validation.test.js b/test/lib/capabilities-echoed-release-hosted-validation.test.js new file mode 100644 index 000000000..0957715f7 --- /dev/null +++ b/test/lib/capabilities-echoed-release-hosted-validation.test.js @@ -0,0 +1,410 @@ +/** + * `get_adcp_capabilities` is validated against the release the agent echoed + * even when an external schema root is active for the pin. + * + * Hosted compliance always runs with an external schema root for the pinned + * release (`comply()` derives it from the compliance folder). A 3.0 pin sends + * only `adcp_major_version: 3`, so a conformant agent serves and echoes its + * highest 3.x release (versioning spec, "Server echoes"). Before the fix the + * pin's external root won over the echo, so every storyboard step's + * `response_schema` check on a 3.2 capabilities response, and the discovery + * notices, validated against 3.0 and failed on fields 3.0 never had. + * + * The echo is used only when it is the pin's major, is listed in the + * response's own `adcp.supported_versions`, and has a schema root (one + * supplied through `servedReleaseSchemaRoots`, an external root registered + * for it, or the SDK's bundled schemas). Otherwise validation stays strictly + * on the pin, and no other task's validation changes. + */ + +const { describe, test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); + +process.env.ADCP_ALLOW_INTERNAL_PROBES = '1'; + +const { ADCPMultiAgentClient } = require('../../dist/lib/index.js'); +const { testCapabilityDiscovery } = require('../../dist/lib/testing/scenarios/capabilities.js'); +const { runValidations } = require('../../dist/lib/testing/storyboard/validations.js'); +const { runStoryboard } = require('../../dist/lib/testing/storyboard/runner.js'); +const { closeMCPConnections } = require('../../dist/lib/protocols/mcp.js'); +const { + resolveServedValidationVersion, + withExternalSchemaRoot, + withServedReleaseSchemaRoots, +} = require('../../dist/lib/validation/schema-loader.js'); +const testing = require('../../dist/lib/testing/index.js'); + +const SCHEMAS_DATA = path.join(__dirname, '../../dist/lib/schemas-data'); +/** Stands in for hosted compliance's external root for a 3.0 target. */ +const PINNED_30_ROOT = path.join(SCHEMAS_DATA, '3.0'); +const MARKER = 'x_hosted_root_marker'; +const CAPABILITIES_FILE = path.join('protocol', 'get-adcp-capabilities-response.json'); +const PRODUCTS_FILE = path.join('media-buy', 'get-products-response.json'); + +/** A conformant AdCP 3.2 capabilities response, invalid against 3.0. */ +function capabilities32(overrides = {}) { + return { + status: 'completed', + adcp_version: '3.2', + adcp: { + major_versions: [3], + supported_versions: ['3.0', '3.1', '3.2'], + idempotency: { supported: true, replay_ttl_seconds: 86400 }, + }, + supported_protocols: ['media_buy'], + account: { require_operator_auth: false, supported_billing: ['operator'] }, + // 3.2 widened `features` entries: 3.0 requires booleans here. + media_buy: { features: { bidding_policy: { media_buy: { fixed: { modes: ['bid_amount'] } } } } }, + // 3.0 closes `scenarios` to six values; 3.1+ opens the list. + compliance_testing: { scenarios: ['force_creative_status', 'force_create_media_buy_arm', 'seed_product'] }, + ...overrides, + }; +} + +function withSupportedVersions(payload, supported_versions) { + return { ...payload, adcp: { ...payload.adcp, supported_versions } }; +} + +/** The 3.0 pin rejects capabilities32() at these pointers. */ +const PIN_30_ISSUE = /must be boolean|must be one of/; + +/** + * A hosted-style 3.2.5 schema root: the SDK's bundled 3.2 capabilities and + * get_products response schemas, each requiring an extra marker field, so a + * test can tell this root apart from the bundled one. + */ +let hosted32Root; +let tmpDir; + +before(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'adcp-echoed-release-')); + hosted32Root = path.join(tmpDir, '3.2.5'); + for (const file of [CAPABILITIES_FILE, PRODUCTS_FILE]) { + const schema = JSON.parse(fs.readFileSync(path.join(SCHEMAS_DATA, '3.2', 'bundled', file), 'utf8')); + schema.required = [...(schema.required ?? []), MARKER]; + const target = path.join(hosted32Root, 'bundled', file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, JSON.stringify(schema)); + } +}); + +after(async () => { + await closeMCPConnections(); + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +const underPinnedRoot = fn => withExternalSchemaRoot('3.0', PINNED_30_ROOT, fn); + +function responseSchemaCheck(taskName, data, extra = {}) { + const [result] = runValidations([{ check: 'response_schema', description: 'response conforms' }], { + taskName, + adcpVersion: '3.0', + responseAdcpVersion: '3.0', + strictResponseSchemaValidation: true, + taskResult: { success: true, data }, + agentUrl: 'https://agent.example/mcp', + contributions: new Set(), + ...extra, + }); + return result; +} + +describe('resolveServedValidationVersion with an external root active for the pin', () => { + test('uses a same-major echo listed in supported_versions and bundled by the SDK', () => { + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(capabilities32(), '3.0')), + '3.2' + ); + }); + + test('keeps the pin when the echo is a different major', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '4.0' }), ['3.0', '4.0']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0' + ); + }); + + test('keeps the pin when the echo is not listed in supported_versions', () => { + const unlisted = withSupportedVersions(capabilities32(), ['3.0', '3.1']); + const missing = capabilities32({ adcp: { major_versions: [3], idempotency: { supported: false } } }); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(unlisted, '3.0')), + '3.0' + ); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(missing, '3.0')), + '3.0' + ); + // Same rule without any external root. + assert.equal(resolveServedValidationVersion(unlisted, '3.0'), '3.0'); + }); + + test('keeps the pin when no schema is available for the echo', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.99' }), ['3.0', '3.99']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0' + ); + }); + + test('counts a served-release root and an external root registered for the echo', () => { + // A release the SDK does not bundle becomes usable once a root is supplied. + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.2' }), ['3.0', '3.2']); + const viaServed = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => resolveServedValidationVersion(payload, '3.0')) + ); + assert.equal(viaServed, '3.2'); + const viaExternal = underPinnedRoot(() => + withExternalSchemaRoot('3.2.5', hosted32Root, () => resolveServedValidationVersion(payload, '3.0')) + ); + assert.equal(viaExternal, '3.2'); + }); +}); + +describe('withServedReleaseSchemaRoots', () => { + test('is exported from @adcp/sdk/testing', () => { + assert.equal(typeof testing.withServedReleaseSchemaRoots, 'function'); + }); + + test('rejects malformed input up front', () => { + assert.throws(() => withServedReleaseSchemaRoots(['3.2'], () => 1), /servedReleaseSchemaRoots/); + assert.throws(() => withServedReleaseSchemaRoots({ '../3.2': hosted32Root }, () => 1), /not a recognized version/); + assert.throws(() => withServedReleaseSchemaRoots({ '3.2.5': '' }, () => 1), /servedReleaseSchemaRoots/); + assert.throws( + () => withServedReleaseSchemaRoots({ '3.2.5': path.join(tmpDir, 'missing') }, () => 1), + /not found or empty/ + ); + // A root whose schema ids belong to another release. + assert.throws(() => withServedReleaseSchemaRoots({ 3.1: hosted32Root }, () => 1), /does not match/); + }); + + test('is a no-op without entries', () => { + assert.equal( + withServedReleaseSchemaRoots(undefined, () => 7), + 7 + ); + assert.equal( + withServedReleaseSchemaRoots({}, () => 8), + 8 + ); + }); +}); + +describe('storyboard response_schema check on get_adcp_capabilities under an external pin root', () => { + test('passes a conformant response for the echoed same-major release', () => { + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', capabilities32())); + assert.equal(result.passed, true, result.error); + }); + + test('still fails a real schema error under the echoed release', () => { + // minProperties: 1 on the 3.2 bidding_policy object. + const payload = capabilities32({ media_buy: { features: { bidding_policy: {} } } }); + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(result.passed, false); + assert.match(result.error, /bidding_policy/); + assert.doesNotMatch(result.error, PIN_30_ISSUE, 'validated against 3.2, not the 3.0 pin'); + }); + + for (const [label, payload] of [ + ['an echo of a different major', withSupportedVersions(capabilities32({ adcp_version: '4.0' }), ['3.0', '4.0'])], + ['an echo not in supported_versions', withSupportedVersions(capabilities32(), ['3.0', '3.1'])], + [ + 'an echo with no schema available', + withSupportedVersions(capabilities32({ adcp_version: '3.99' }), ['3.0', '3.99']), + ], + ['no echo', (({ adcp_version: _omitted, ...rest }) => rest)(capabilities32())], + ]) { + test(`falls back to the pin, strictly, for ${label}`, () => { + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(result.passed, false, 'the 3.0 pin rejects 3.2 shapes'); + assert.match(result.error, PIN_30_ISSUE); + }); + } + + test('uses the root supplied for the echoed release', () => { + const missingMarker = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + responseSchemaCheck('get_adcp_capabilities', capabilities32()) + ) + ); + assert.equal(missingMarker.passed, false); + assert.match(missingMarker.error, new RegExp(MARKER)); + + const withMarker = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + responseSchemaCheck('get_adcp_capabilities', capabilities32({ [MARKER]: true })) + ) + ); + assert.equal(withMarker.passed, true, withMarker.error); + }); + + test('does not change other tasks: they validate against the pin and ignore served roots', () => { + // A get_products response that echoes 3.2 and lists it is still graded + // against the pinned 3.0 root, never the served 3.2 root. + const products = { + adcp_version: '3.2', + adcp: { supported_versions: ['3.0', '3.2'] }, + products: [], + }; + const baseline = underPinnedRoot(() => responseSchemaCheck('get_products', { products: [] })); + const echoed = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => responseSchemaCheck('get_products', products)) + ); + assert.equal(echoed.passed, baseline.passed, echoed.error); + assert.equal(echoed.schema_id, baseline.schema_id); + assert.doesNotMatch(echoed.error ?? '', new RegExp(MARKER)); + }); +}); + +// ── Live seller: discovery notices, TaskExecutor and runStoryboard ────────── + +async function startSeller({ payload }) { + const { createMcpHandler, McpServer } = require('@modelcontextprotocol/server'); + const { toNodeHandler } = require('@modelcontextprotocol/node'); + const { z } = require('zod'); + const handler = createMcpHandler(() => { + const server = new McpServer({ name: 'echoed-release-seller', version: '1.0.0' }); + server.registerTool( + 'get_adcp_capabilities', + { description: 'AdCP capabilities', inputSchema: z.object({}).passthrough() }, + async () => ({ content: [{ type: 'text', text: JSON.stringify(payload) }], structuredContent: payload }) + ); + for (const name of ['get_products', 'create_media_buy', 'comply_test_controller']) { + server.registerTool(name, { description: name }, async () => ({ + content: [{ type: 'text', text: '{}' }], + structuredContent: {}, + })); + } + return server; + }); + const serveMcp = toNodeHandler(handler); + const server = http.createServer(async (req, res) => { + if (req.url !== '/mcp' && req.url !== '/mcp/') { + res.writeHead(404).end('not found'); + return; + } + await serveMcp(req, res); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { + url: `http://127.0.0.1:${server.address().port}/mcp`, + async close() { + await handler.close(); + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + }, + }; +} + +async function withSeller(t, payload) { + await closeMCPConnections(); + const seller = await startSeller({ payload }); + t.after(async () => { + await closeMCPConnections(); + await seller.close(); + }); + return seller; +} + +function capabilitiesStoryboard() { + return { + id: 'echoed_release_capabilities', + version: '1.0.0', + title: 'Echoed-release capabilities', + category: 'test', + summary: 'A single get_adcp_capabilities step with a response_schema check.', + narrative: '', + agent: { interaction_model: 'sync', capabilities: [] }, + caller: { role: 'buyer_agent' }, + phases: [ + { + id: 'capabilities', + title: 'Capabilities', + steps: [ + { + id: 'discover', + title: 'Discover capabilities', + task: 'get_adcp_capabilities', + response_schema_ref: 'protocol/get-adcp-capabilities-response.json', + validations: [{ check: 'response_schema', description: 'response conforms' }], + }, + ], + }, + ], + }; +} + +describe('live seller serving 3.2 to a 3.0 pin with an external pin root', () => { + test('discovery reports no capability schema issues for a conformant 3.2 response', async t => { + const seller = await withSeller(t, capabilities32()); + const discovery = await underPinnedRoot(() => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ); + assert.equal(discovery.profile?.capabilities_schema_issues, undefined); + }); + + test('discovery validates against the supplied root for the echoed release', async t => { + const seller = await withSeller(t, capabilities32()); + const discovery = await underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ) + ); + const issues = discovery.profile?.capabilities_schema_issues ?? []; + assert.ok( + issues.some(issue => issue.pointer === `/${MARKER}` || issue.message.includes(MARKER)), + JSON.stringify(issues) + ); + assert.ok( + issues.every(issue => !PIN_30_ISSUE.test(issue.message)), + JSON.stringify(issues) + ); + }); + + test('discovery keeps strict pin validation for an echo not in supported_versions', async t => { + const seller = await withSeller(t, withSupportedVersions(capabilities32(), ['3.0', '3.1'])); + const discovery = await underPinnedRoot(() => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ); + const issues = discovery.profile?.capabilities_schema_issues ?? []; + assert.ok( + issues.some(issue => PIN_30_ISSUE.test(issue.message)), + JSON.stringify(issues) + ); + }); + + test('TaskExecutor accepts getAdcpCapabilities in strict mode', async t => { + const seller = await withSeller(t, capabilities32()); + const client = new ADCPMultiAgentClient( + [{ id: 'seller', name: 'Seller', agent_uri: seller.url, protocol: 'mcp' }], + { adcpVersion: '3.0', validation: { responses: 'strict', logSchemaViolations: false } } + ); + const result = await underPinnedRoot(() => client.agent('seller').getAdcpCapabilities({})); + assert.equal(result.success, true, result.error); + }); + + test('runStoryboard passes the capabilities step and honors servedReleaseSchemaRoots', async t => { + const seller = await withSeller(t, capabilities32()); + const options = { protocol: 'mcp', adcpVersion: '3.0', schemaRoot: PINNED_30_ROOT }; + + const passed = await runStoryboard(seller.url, capabilitiesStoryboard(), options); + const step = passed.phases[0].steps[0]; + const check = step.validations.find(v => v.check === 'response_schema'); + assert.equal(check?.passed, true, JSON.stringify(step, null, 2)); + + await closeMCPConnections(); + const served = await runStoryboard(seller.url, capabilitiesStoryboard(), { + ...options, + servedReleaseSchemaRoots: { '3.2.5': hosted32Root }, + }); + const servedStep = served.phases[0].steps[0]; + const servedCheck = servedStep.validations.find(v => v.check === 'response_schema'); + assert.equal(servedCheck?.passed, false, JSON.stringify(servedStep, null, 2)); + assert.match(servedCheck.error, new RegExp(MARKER)); + }); +}); diff --git a/test/lib/capabilities-served-release-validation.test.js b/test/lib/capabilities-served-release-validation.test.js index 77444c780..1a4900dbd 100644 --- a/test/lib/capabilities-served-release-validation.test.js +++ b/test/lib/capabilities-served-release-validation.test.js @@ -106,55 +106,63 @@ async function withSeller(t, options) { const stepNamed = (steps, name) => steps.find(step => step.step === name); const failedSteps = steps => steps.filter(step => !step.passed && !step.skipped); +/** An echo of `release` that the response also lists in `supported_versions`. */ +const echo = (release, supported = ['3.0', '3.1', '3.2', release]) => ({ + adcp_version: release, + adcp: { supported_versions: [...new Set(supported)] }, +}); + describe('resolveServedValidationVersion', () => { - test('uses a bundled same-major echo instead of the pin', () => { - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.2'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.1' }, '3.0.25'), '3.1'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.0' }, '3.2'), '3.0'); + test('uses a bundled same-major echo listed in supported_versions instead of the pin', () => { + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.0'), '3.2'); + assert.equal(resolveServedValidationVersion(echo('3.1'), '3.0.25'), '3.1'); + assert.equal(resolveServedValidationVersion(echo('3.0'), '3.2'), '3.0'); }); test('keeps the exact pin when the echo is the pinned release', () => { // Preserves a patch-precise pin (and any external schema root bound to it). - assert.equal(resolveServedValidationVersion({ adcp_version: '3.0' }, '3.0.25'), '3.0.25'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.2'), '3.2'); + assert.equal(resolveServedValidationVersion(echo('3.0'), '3.0.25'), '3.0.25'); + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.2'), '3.2'); }); test('falls back to the pin when there is no usable echo', () => { assert.equal(resolveServedValidationVersion({}, '3.0'), '3.0', 'no echo'); assert.equal(resolveServedValidationVersion(undefined, '3.0'), '3.0', 'no payload'); - assert.equal(resolveServedValidationVersion({ adcp_version: 3.2 }, '3.0'), '3.0', 'non-string echo'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2.4' }, '3.0'), '3.0', 'not release precision'); - assert.equal(resolveServedValidationVersion({ adcp_version: '../3.2' }, '3.0'), '3.0', 'garbage'); - assert.equal(resolveServedValidationVersion({ adcp_version: '4.0' }, '3.0'), '3.0', 'different major'); - assert.equal(resolveServedValidationVersion({ adcp_version: '2.5' }, '3.0'), '3.0', 'older major'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.99' }, '3.0'), '3.0', 'no bundled schema'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, 'v2.5'), 'v2.5', 'legacy v2 pin'); + assert.equal(resolveServedValidationVersion({ ...echo('3.2'), adcp_version: 3.2 }, '3.0'), '3.0', 'non-string'); + assert.equal(resolveServedValidationVersion(echo('3.2.4'), '3.0'), '3.0', 'not release precision'); + assert.equal(resolveServedValidationVersion(echo('../3.2'), '3.0'), '3.0', 'garbage'); + assert.equal(resolveServedValidationVersion(echo('4.0'), '3.0'), '3.0', 'different major'); + assert.equal(resolveServedValidationVersion(echo('2.5'), '3.0'), '3.0', 'older major'); + assert.equal(resolveServedValidationVersion(echo('3.99'), '3.0'), '3.0', 'no bundled schema'); + assert.equal(resolveServedValidationVersion(echo('3.2'), 'v2.5'), 'v2.5', 'legacy v2 pin'); }); - test('does not consult supported_versions', () => { - // Spec: with no echo, validate against the pin. + test('requires the echo to be listed in supported_versions', () => { + assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.0', 'no supported_versions'); + assert.equal(resolveServedValidationVersion(echo('3.2', ['3.0', '3.1']), '3.0'), '3.0', 'not listed'); + assert.equal( + resolveServedValidationVersion({ adcp_version: '3.2', adcp: { supported_versions: '3.2' } }, '3.0'), + '3.0', + 'malformed supported_versions' + ); + // supported_versions alone never moves validation off the pin. assert.equal(resolveServedValidationVersion({ adcp: { supported_versions: ['3.0', '3.2'] } }, '3.0'), '3.0'); }); test('handles prerelease and legacy pins', () => { - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.1-beta'), '3.2', 'stable echo, beta pin'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.1-beta.1' }, '3.1.0-beta.1'), '3.1.0-beta.1'); - assert.equal( - resolveServedValidationVersion({ adcp_version: '3.4-rc.1' }, '3.0'), - '3.0', - 'unbundled prerelease echo' - ); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, 'v3'), 'v3', 'legacy alias pin'); + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.1-beta'), '3.2', 'stable echo, beta pin'); + assert.equal(resolveServedValidationVersion(echo('3.1-beta.1'), '3.1.0-beta.1'), '3.1.0-beta.1'); + assert.equal(resolveServedValidationVersion(echo('3.4-rc.1'), '3.0'), '3.0', 'unbundled prerelease echo'); + assert.equal(resolveServedValidationVersion(echo('3.2'), 'v3'), 'v3', 'legacy alias pin'); }); - test('an external schema root bound to the pin wins over the echo', () => { - // Same precedence as the storyboard runner's response_schema check. + test('an external schema root bound to the pin does not override a usable echo', () => { + // Hosted compliance always runs with one (adcp-client#3320). const root = path.join(__dirname, '../../dist/lib/schemas-data/3.0'); - const selected = withExternalSchemaRoot('3.0', root, () => - resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0') - ); - assert.equal(selected, '3.0'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.2', 'scope ended'); + const selected = withExternalSchemaRoot('3.0', root, () => resolveServedValidationVersion(echo('3.2'), '3.0')); + assert.equal(selected, '3.2'); + const pinnedEcho = withExternalSchemaRoot('3.0', root, () => resolveServedValidationVersion(echo('3.0'), '3.0')); + assert.equal(pinnedEcho, '3.0', 'an echo of the pinned release keeps the pin and its root'); }); }); From 08574ce05917565606d96c220002d18824b69b99 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 17 Oct 2026 13:24:51 +0000 Subject: [PATCH 2/2] fix(testing): honor served roots in the runner's Zod recovery path; reject conflicting served-root keys Address review: the runner's Zod-rejection recovery treats a capabilities response validated against an echoed release with an external or supplied root as authoritative; withServedReleaseSchemaRoots rejects two keys for the same release with different roots; document the pin requirement and root lifetime. Tests now cover an unbundled echoed release made usable by a supplied root, served roots never activating for other tasks, and the hosted shape end to end through comply(). Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/testing/storyboard/validations.ts | 15 ++- src/lib/testing/types.ts | 3 + src/lib/validation/schema-loader.ts | 11 +- ...s-echoed-release-hosted-validation.test.js | 105 ++++++++++++++++-- 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index 79b8f7bb4..c142c46f2 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -719,10 +719,21 @@ function validateResponseSchemaForVersion( /** @internal Shared with the storyboard runner's Zod-rejection recovery path. */ export function isExternalResponseSchemaAuthoritative( - ctx: Pick + ctx: Pick & + Partial> ): boolean { const validationVersion = responseSchemaValidationVersion(ctx.adcpVersion, ctx.responseAdcpVersion); - return validationVersion !== undefined && isExternalSchemaRootActive(validationVersion); + if (validationVersion !== undefined && isExternalSchemaRootActive(validationVersion)) return true; + // A capabilities response validated against its echoed release is + // authoritative when that release's root is external (registered, or + // supplied through servedReleaseSchemaRoots), same as a pinned external root. + const pinnedVersion = ctx.adcpVersion; + if (ctx.taskName !== 'get_adcp_capabilities' || pinnedVersion === undefined || !ctx.taskResult) return false; + return withServedValidationVersion( + ctx.taskResult.data, + pinnedVersion, + version => version !== pinnedVersion && isExternalSchemaRootActive(version) + ); } function responseSchemaValidationVersion( diff --git a/src/lib/testing/types.ts b/src/lib/testing/types.ts index 5679feec4..0c1c38920 100644 --- a/src/lib/testing/types.ts +++ b/src/lib/testing/types.ts @@ -148,6 +148,9 @@ export interface TestOptions { * root is checked against its key once per process; a malformed map throws * before the run starts. Honored by `comply()`, `runStoryboard()` and * `runStoryboardStep()`; other callers can use `withServedReleaseSchemaRoots`. + * The echo is compared with the run's pin, so storyboard checks apply it + * only when the run has one (`adcpVersion`, or the storyboard or compliance + * bundle's `adcp_version`); `comply()` always does. * * @example * ```ts diff --git a/src/lib/validation/schema-loader.ts b/src/lib/validation/schema-loader.ts index 4ada3c43d..470cdf4ef 100644 --- a/src/lib/validation/schema-loader.ts +++ b/src/lib/validation/schema-loader.ts @@ -1296,7 +1296,9 @@ const servedReleaseSchemaRootScope = new AsyncLocalStorage>(); @@ -1345,6 +1347,13 @@ export function withServedReleaseSchemaRoots(roots: Readonly entry.root !== root && [...aliases].some(alias => entry.aliases.has(alias))); + if (conflict) { + throw new Error( + `servedReleaseSchemaRoots["${version}"] and servedReleaseSchemaRoots["${conflict.version}"] name the same ` + + `release with different schema roots (${root}, ${conflict.root})` + ); + } entries.push({ version, root, aliases }); } if (entries.length === 0) return fn(); diff --git a/test/lib/capabilities-echoed-release-hosted-validation.test.js b/test/lib/capabilities-echoed-release-hosted-validation.test.js index 0957715f7..c64a90272 100644 --- a/test/lib/capabilities-echoed-release-hosted-validation.test.js +++ b/test/lib/capabilities-echoed-release-hosted-validation.test.js @@ -32,6 +32,7 @@ const { runValidations } = require('../../dist/lib/testing/storyboard/validation const { runStoryboard } = require('../../dist/lib/testing/storyboard/runner.js'); const { closeMCPConnections } = require('../../dist/lib/protocols/mcp.js'); const { + isExternalSchemaRootActive, resolveServedValidationVersion, withExternalSchemaRoot, withServedReleaseSchemaRoots, @@ -78,6 +79,8 @@ const PIN_30_ISSUE = /must be boolean|must be one of/; * test can tell this root apart from the bundled one. */ let hosted32Root; +/** A root for 3.9, a release the SDK does not bundle (3.2 shapes, 3.9 ids). */ +let unbundled39Root; let tmpDir; before(() => { @@ -90,6 +93,13 @@ before(() => { fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, JSON.stringify(schema)); } + unbundled39Root = path.join(tmpDir, '3.9.0'); + const capabilities = fs + .readFileSync(path.join(SCHEMAS_DATA, '3.2', 'bundled', CAPABILITIES_FILE), 'utf8') + .replaceAll('/schemas/3.2.5/', '/schemas/3.9.0/'); + const target39 = path.join(unbundled39Root, 'bundled', CAPABILITIES_FILE); + fs.mkdirSync(path.dirname(target39), { recursive: true }); + fs.writeFileSync(target39, capabilities); }); after(async () => { @@ -152,17 +162,21 @@ describe('resolveServedValidationVersion with an external root active for the pi ); }); - test('counts a served-release root and an external root registered for the echo', () => { - // A release the SDK does not bundle becomes usable once a root is supplied. - const payload = withSupportedVersions(capabilities32({ adcp_version: '3.2' }), ['3.0', '3.2']); + test('an unbundled echo becomes usable once a served or external root is supplied for it', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.9' }), ['3.0', '3.2', '3.9']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0', + 'no root: pin' + ); const viaServed = underPinnedRoot(() => - withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => resolveServedValidationVersion(payload, '3.0')) + withServedReleaseSchemaRoots({ '3.9.0': unbundled39Root }, () => resolveServedValidationVersion(payload, '3.0')) ); - assert.equal(viaServed, '3.2'); + assert.equal(viaServed, '3.9'); const viaExternal = underPinnedRoot(() => - withExternalSchemaRoot('3.2.5', hosted32Root, () => resolveServedValidationVersion(payload, '3.0')) + withExternalSchemaRoot('3.9.0', unbundled39Root, () => resolveServedValidationVersion(payload, '3.0')) ); - assert.equal(viaExternal, '3.2'); + assert.equal(viaExternal, '3.9'); }); }); @@ -181,6 +195,11 @@ describe('withServedReleaseSchemaRoots', () => { ); // A root whose schema ids belong to another release. assert.throws(() => withServedReleaseSchemaRoots({ 3.1: hosted32Root }, () => 1), /does not match/); + // Two keys for the same release with different roots. + assert.throws( + () => withServedReleaseSchemaRoots({ '3.2.5': hosted32Root, 3.2: path.join(SCHEMAS_DATA, '3.2') }, () => 1), + /same release with different schema roots/ + ); }); test('is a no-op without entries', () => { @@ -243,6 +262,20 @@ describe('storyboard response_schema check on get_adcp_capabilities under an ext assert.equal(withMarker.passed, true, withMarker.error); }); + test('validates an unbundled echo against the root supplied for it', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.9' }), ['3.0', '3.9']); + const unsupplied = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(unsupplied.passed, false, 'no 3.9 schema: strict pin'); + assert.match(unsupplied.error, PIN_30_ISSUE); + + const supplied = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.9.0': unbundled39Root }, () => + responseSchemaCheck('get_adcp_capabilities', payload) + ) + ); + assert.equal(supplied.passed, true, supplied.error); + }); + test('does not change other tasks: they validate against the pin and ignore served roots', () => { // A get_products response that echoes 3.2 and lists it is still graded // against the pinned 3.0 root, never the served 3.2 root. @@ -259,6 +292,25 @@ describe('storyboard response_schema check on get_adcp_capabilities under an ext assert.equal(echoed.schema_id, baseline.schema_id); assert.doesNotMatch(echoed.error ?? '', new RegExp(MARKER)); }); + + test('served roots never become active for other tasks, even on the served release', () => { + // Pin 3.2 with a supplied 3.2.5 root: a get_products response is still + // validated against the SDK's bundled 3.2, which has no marker field. + const extra = { + adcpVersion: '3.2', + responseAdcpVersion: '3.2', + responseSchemaRef: 'media-buy/get-products-response.json', + }; + const baseline = responseSchemaCheck('get_products', { products: [] }, extra); + const scoped = withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => { + assert.equal(isExternalSchemaRootActive('3.2'), false, 'scope does not register the root'); + return responseSchemaCheck('get_products', { products: [] }, extra); + }); + assert.equal(scoped.passed, baseline.passed, scoped.error); + assert.ok(scoped.schema_id, 'schema identity resolved'); + assert.equal(scoped.schema_id, baseline.schema_id); + assert.doesNotMatch(scoped.error ?? '', new RegExp(MARKER)); + }); }); // ── Live seller: discovery notices, TaskExecutor and runStoryboard ────────── @@ -340,6 +392,45 @@ function capabilitiesStoryboard() { } describe('live seller serving 3.2 to a 3.0 pin with an external pin root', () => { + test('comply() with a 3.0 compliance cache and pin root grades the echoed 3.2 capabilities', async t => { + // Hosted compliance shape: comply() opens the external 3.0 root itself. + const seller = await withSeller(t, capabilities32()); + const run = extra => + testing.comply(seller.url, { + version: '3.0.25', + schemaRoot: PINNED_30_ROOT, + storyboards: ['capability_discovery'], + protocol: 'mcp', + allow_http: true, + ...extra, + }); + const schemaFailures = result => (result.failures ?? []).filter(f => f.validation?.check === 'response_schema'); + const capabilityNotices = result => + (result.notices ?? []).filter(n => n.code === 'capabilities_response_schema_invalid'); + + const result = await run({}); + assert.deepEqual(schemaFailures(result), [], JSON.stringify(schemaFailures(result), null, 2)); + assert.deepEqual(capabilityNotices(result), []); + + await closeMCPConnections(); + const served = await run({ servedReleaseSchemaRoots: { '3.2.5': hosted32Root } }); + const failures = schemaFailures(served); + assert.ok(failures.length > 0, 'the supplied 3.2.5 root grades the capabilities steps'); + assert.ok( + failures.every(f => JSON.stringify(f).includes(MARKER)), + JSON.stringify(failures, null, 2) + ); + assert.ok( + capabilityNotices(served).some(n => JSON.stringify(n).includes(MARKER)), + JSON.stringify(served.notices, null, 2) + ); + + await assert.rejects( + run({ servedReleaseSchemaRoots: { '3.2.5': path.join(tmpDir, 'missing') } }), + /not found or empty/ + ); + }); + test('discovery reports no capability schema issues for a conformant 3.2 response', async t => { const seller = await withSeller(t, capabilities32()); const discovery = await underPinnedRoot(() =>