diff --git a/.changeset/capabilities-echoed-release-external-root.md b/.changeset/capabilities-echoed-release-external-root.md new file mode 100644 index 000000000..040bf66cd --- /dev/null +++ b/.changeset/capabilities-echoed-release-external-root.md @@ -0,0 +1,9 @@ +--- +'@adcp/sdk': patch +--- + +Validate `get_adcp_capabilities` against the release the agent echoed even when an external schema root is active for the pin (#3320). Hosted compliance always runs with one, so a 3.0 target graded a conformant 3.2 capabilities response against 3.0: every storyboard `response_schema` check on that task failed, and discovery raised spurious `capabilities_response_schema_invalid` notices. + +The storyboard `response_schema` check, discovery's capability validation and `getAdcpCapabilities()` now use the echoed `adcp_version` when it is the pin's major, is listed in the response's own `adcp.supported_versions`, and has a schema root: one supplied for that release, an external root registered for it, or the SDK's bundled schemas. Otherwise they validate strictly against the pin. Requiring the `supported_versions` listing is new. Other tasks, storyboard selection, the run's `adcp_version` and grading stay on the pin. + +New `servedReleaseSchemaRoots` option on `comply()`, `runStoryboard()` and `runStoryboardStep()`: a map of AdCP version to schema root (same shape as `schemaRoot`) for releases an agent may echo. Those roots are used only for this capabilities validation. `withServedReleaseSchemaRoots(roots, fn)` from `@adcp/sdk/testing` scopes the same map for other callers. diff --git a/docs/guides/VALIDATE-YOUR-AGENT.md b/docs/guides/VALIDATE-YOUR-AGENT.md index a947ec48c..dde300d4f 100644 --- a/docs/guides/VALIDATE-YOUR-AGENT.md +++ b/docs/guides/VALIDATE-YOUR-AGENT.md @@ -187,6 +187,40 @@ await comply(agentUrl, { }); ``` +**Capabilities served at a newer release.** A 3.0 pin sends only +`adcp_major_version: 3`, so a conformant agent may serve and echo its highest +3.x release in `get_adcp_capabilities` (versioning spec, "Server echoes"). The +runner validates that one response against the echoed release when the echo is +the pin's major, is listed in the response's own `adcp.supported_versions`, and +has a schema root. That covers discovery's `capabilities_response_schema_invalid` +notices, `getAdcpCapabilities()` and every storyboard `response_schema` check on +`get_adcp_capabilities`, even with `schemaRoot` set for the pin. Otherwise it +validates strictly against the pin. Every other task's response, storyboard +selection, the run's `adcp_version` and grading stay on the pin. + +The echoed release's schemas come from the SDK's bundle by default. A grader +that ships its own roots per release can supply them with +`servedReleaseSchemaRoots` (on `comply()`, `runStoryboard()` and +`runStoryboardStep()`), keyed by AdCP version: + +```ts +await comply(agentUrl, { + version: '3.0.27', + complianceDir: '/app/dist/compliance/3.0.27', + schemaRoot: '/app/dist/schemas/3.0.27', + servedReleaseSchemaRoots: { + '3.1.29': '/app/dist/schemas/3.1.29', + '3.2.5': '/app/dist/schemas/3.2.5', + }, +}); +``` + +Those roots are used only for that capabilities validation. Each one is checked +against its key once per process, and a mismatched or missing root throws +before the run starts. Callers that validate outside `comply()` and the runner +can scope the same map with `withServedReleaseSchemaRoots(roots, fn)` from +`@adcp/sdk/testing`. + **Multi-agent storyboards under `comply()`.** `comply()` grades one agent, so storyboards that declare `requires: [multi_agent]` (for example the governance-aware seller scenarios) skip with `requirement_unmet` and cap their diff --git a/src/lib/core/TaskExecutor.ts b/src/lib/core/TaskExecutor.ts index a7fc4fcd7..fb73b18f0 100644 --- a/src/lib/core/TaskExecutor.ts +++ b/src/lib/core/TaskExecutor.ts @@ -34,7 +34,7 @@ import { type ValidationMode, } from '../validation/client-hooks'; import { formatIssues } from '../validation/schema-validator'; -import { resolveServedValidationVersion } from '../validation/schema-loader'; +import { withServedValidationVersion } from '../validation/schema-loader'; import { ADCP_VERSION } from '../version'; import { unwrapProtocolResponse, isAdcpOperationSuccess } from '../utils/response-unwrapper'; import { extractAdcpErrorInfo, extractCorrelationId } from '../utils/error-extraction'; @@ -6125,64 +6125,31 @@ export class TaskExecutor { // Normal dispatch uses lastKnownServerVersion; restart recovery passes // the persisted continuation version explicitly so concurrent or fresh // executors cannot validate the seller's response against another wire. - let validationVersion = + const pinnedValidationVersion = (serverVersion ?? this.lastKnownServerVersion) === 'v2' ? 'v2.5' : (this.responseAdcpVersionForValidation() ?? this.config.adcpVersion ?? ADCP_VERSION); // Capabilities are negotiated before any pin applies: a 3.0 pin (or a // major-only envelope) is served the seller's highest release in the // major, echoed in `adcp_version`. Validate against that release, as the - // versioning spec requires ("Server echoes"), falling back to the pin - // when there is no usable echo. Only the validator's version changes; - // the unwrapper keeps `effectiveResponseAdcpVersion()`. - if (taskName === 'get_adcp_capabilities' && validationVersion !== 'v2.5') { - validationVersion = resolveServedValidationVersion(response, validationVersion); - } - const normalizedResponse = this.normalizeResponseForValidation(response, taskName, validationVersion === 'v2.5'); - // TaskExecutor owns the user-facing log entry below. Passing debugLogs - // into the lower-level hook would emit a second, unversioned duplicate. - const outcome = - (this.config.experimentalIdentity33?.() || - (this.config.identity33SchemaExperiment && - normalizedResponse && - typeof normalizedResponse === 'object' && - isAdcpVersionAtLeast((normalizedResponse as { adcp_version?: string }).adcp_version, '3.3'))) && - mode !== 'off' - ? validateIdentity33Payload(taskName, 'response', normalizedResponse, validationVersion) - : validateIncomingResponse(taskName, normalizedResponse, mode, undefined, validationVersion); - if (outcome.valid) return { valid: true, errors: [] }; - - const errorStrings = outcome.issues.map(i => `${i.pointer}: ${i.message}`); - - if (logViolations) { - debugLogs.push({ - timestamp: new Date().toISOString(), - type: mode === 'strict' ? 'error' : 'warning', - message: `Schema validation ${mode === 'strict' ? 'failed' : 'warning'} for ${taskName}: ${formatIssues( - outcome.issues - )}`, - errors: errorStrings, - issues: outcome.issues, - schemaVariant: outcome.variant, - schemaVersion: validationVersion, - mode, - }); - } - - if (mode === 'warn') { - if (logViolations) { - console.warn( - `Schema validation failed for ${taskName} against AdCP ${validationVersion} (non-blocking):`, - errorStrings - ); - } - return { valid: true, errors: [] }; - } - - if (logViolations) { - console.error(`Schema validation failed for ${taskName} against AdCP ${validationVersion}:`, errorStrings); + // versioning spec requires ("Server echoes"), when it is same-major, + // listed in the response's `supported_versions` and has a schema root; + // otherwise against the pin. This holds even when an external schema + // root is active for the pin. Only the validator's version changes; the + // unwrapper keeps `effectiveResponseAdcpVersion()`. + if (taskName === 'get_adcp_capabilities' && pinnedValidationVersion !== 'v2.5') { + return withServedValidationVersion(response, pinnedValidationVersion, validationVersion => + this.validateResponseSchemaAgainst(response, taskName, debugLogs, mode, logViolations, validationVersion) + ); } - return { valid: false, errors: errorStrings }; + return this.validateResponseSchemaAgainst( + response, + taskName, + debugLogs, + mode, + logViolations, + pinnedValidationVersion + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'unknown error'; console.error('Error during schema validation:', errorMessage); @@ -6193,6 +6160,61 @@ export class TaskExecutor { } } + private validateResponseSchemaAgainst( + response: any, + taskName: string, + debugLogs: any[], + mode: ValidationMode, + logViolations: boolean, + validationVersion: string + ): { valid: boolean; errors: string[] } { + const normalizedResponse = this.normalizeResponseForValidation(response, taskName, validationVersion === 'v2.5'); + // TaskExecutor owns the user-facing log entry below. Passing debugLogs + // into the lower-level hook would emit a second, unversioned duplicate. + const outcome = + (this.config.experimentalIdentity33?.() || + (this.config.identity33SchemaExperiment && + normalizedResponse && + typeof normalizedResponse === 'object' && + isAdcpVersionAtLeast((normalizedResponse as { adcp_version?: string }).adcp_version, '3.3'))) && + mode !== 'off' + ? validateIdentity33Payload(taskName, 'response', normalizedResponse, validationVersion) + : validateIncomingResponse(taskName, normalizedResponse, mode, undefined, validationVersion); + if (outcome.valid) return { valid: true, errors: [] }; + + const errorStrings = outcome.issues.map(i => `${i.pointer}: ${i.message}`); + + if (logViolations) { + debugLogs.push({ + timestamp: new Date().toISOString(), + type: mode === 'strict' ? 'error' : 'warning', + message: `Schema validation ${mode === 'strict' ? 'failed' : 'warning'} for ${taskName}: ${formatIssues( + outcome.issues + )}`, + errors: errorStrings, + issues: outcome.issues, + schemaVariant: outcome.variant, + schemaVersion: validationVersion, + mode, + }); + } + + if (mode === 'warn') { + if (logViolations) { + console.warn( + `Schema validation failed for ${taskName} against AdCP ${validationVersion} (non-blocking):`, + errorStrings + ); + } + return { valid: true, errors: [] }; + } + + if (logViolations) { + console.error(`Schema validation failed for ${taskName} against AdCP ${validationVersion}:`, errorStrings); + } + return { valid: false, errors: errorStrings }; + } + private responseAdcpVersionForValidation(): string | undefined { if (this.config.versionEnvelope === 'major-only') return '3.0'; return undefined; diff --git a/src/lib/testing/client.ts b/src/lib/testing/client.ts index bf9b411b0..32918a463 100644 --- a/src/lib/testing/client.ts +++ b/src/lib/testing/client.ts @@ -28,7 +28,7 @@ import type { AgentRequestSigningConfig } from '../types/adcp'; import { createHmac, hkdfSync, randomBytes } from 'node:crypto'; import { AsyncLocalStorage } from 'node:async_hooks'; import { validateIncomingResponse } from '../validation/client-hooks'; -import { resolveServedValidationVersion } from '../validation/schema-loader'; +import { withServedValidationVersion } from '../validation/schema-loader'; import { extractVersionUnsupportedDetails } from '../utils/error-extraction'; import { buildAgentSigningContext } from '../signing/client'; import type { VerifierCapability } from '../signing/types'; @@ -848,13 +848,14 @@ export async function discoverAgentProfile( // Validate against the release the seller served (its echoed // `adcp_version`), not the pin. A 3.0 pin negotiates major-only, so // the seller serves its highest 3.x release (versioning spec, - // "Server echoes"). Falls back to the pin when there is no usable echo. - const validation = validateIncomingResponse( - 'get_adcp_capabilities', + // "Server echoes"). The echo is used when it is same-major, listed in + // the response's `supported_versions` and has a schema root, even + // with an external root active for the pin; otherwise the pin. + const validation = withServedValidationVersion( caps.data, - 'strict', - undefined, - resolveServedValidationVersion(caps.data, schemaAdcpVersion ?? client.getAdcpVersion()) + schemaAdcpVersion ?? client.getAdcpVersion(), + validationVersion => + validateIncomingResponse('get_adcp_capabilities', caps.data, 'strict', undefined, validationVersion) ); if (!validation.valid) { profile.capabilities_schema_issues = validation.issues.map(issue => ({ diff --git a/src/lib/testing/compliance/comply.ts b/src/lib/testing/compliance/comply.ts index 166f3efe7..4b455fa7a 100644 --- a/src/lib/testing/compliance/comply.ts +++ b/src/lib/testing/compliance/comply.ts @@ -80,7 +80,7 @@ import { detectController, hasTestController } from '../test-controller'; import type { ControllerDetection } from '../test-controller'; import { randomBytes } from 'crypto'; import { isPre31AdcpVersion } from '../../utils/adcp-version-config'; -import { withExternalSchemaRoot } from '../../validation/schema-loader'; +import { withExternalSchemaRoot, withServedReleaseSchemaRoots } from '../../validation/schema-loader'; import { redactOAuthUrlForOutput, redactOAuthUrlsInText } from '../storyboard/oauth-metadata-graph'; import { LIBRARY_VERSION } from '../../version'; import { validationFailsStep } from '../storyboard/validations'; @@ -2287,7 +2287,13 @@ async function complyImpl(agentUrl: string, options: ComplyOptions): Promise { + // `servedReleaseSchemaRoots` stays in `testOptions` so routed and nested + // storyboard runs see it too; scoping it here also covers discovery. + return await withExternalSchemaRoot(complianceIndex.adcp_version, scopedSchemaRoot, () => + withServedReleaseSchemaRoots(testOptions.servedReleaseSchemaRoots, () => complyInSchemaScope()) + ); + + async function complyInSchemaScope(): Promise { let effectiveOptions: TestOptions = applyAdcpVersionRunOptions(complianceIndex.adcp_version, { ...testOptions, // `comply()` has always defaulted an unset transport to MCP when it built @@ -2725,7 +2731,7 @@ async function complyImpl(agentUrl: string, options: ComplyOptions): Promise + const runBody = () => + withServedReleaseSchemaRoots(options.servedReleaseSchemaRoots, () => runStoryboardBody(agentUrlOrUrls, storyboard, options) ); + if (schemaRoot) { + return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, runBody); } - return await runStoryboardBody(agentUrlOrUrls, storyboard, options); + return await runBody(); }, { isolate: !hasTestClientScope() } ); @@ -5489,12 +5492,14 @@ export async function runStoryboardStep( options = resolveDeclaredTestKit(storyboard, options); options = applyNativeA2AComplianceTransportOptions(options); const schemaRoot = getRunSchemaRoot(options); - if (schemaRoot) { - return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, () => + const runBody = () => + withServedReleaseSchemaRoots(options.servedReleaseSchemaRoots, () => runStoryboardStepBody(agentUrl, storyboard, stepId, options) ); + if (schemaRoot) { + return await withExternalSchemaRoot(schemaRoot.adcpVersion, schemaRoot.schemaRoot, runBody); } - return await runStoryboardStepBody(agentUrl, storyboard, stepId, options); + return await runBody(); }, { isolate: !hasTestClientScope() } ); diff --git a/src/lib/testing/storyboard/validations.ts b/src/lib/testing/storyboard/validations.ts index 4e0c2a5e4..c142c46f2 100644 --- a/src/lib/testing/storyboard/validations.ts +++ b/src/lib/testing/storyboard/validations.ts @@ -12,7 +12,11 @@ import { prepareResponseForSchemaValidation, TOOL_RESPONSE_SCHEMAS } from '../.. import { injectLegacyEnvelopeStatus } from '../../utils/envelope-status-compat'; import { TRANSPORT_SUFFIX_REGEX } from '../../utils/a2a-discovery'; import { validateResponse, type ValidationIssue } from '../../validation/schema-validator'; -import { hasSchemaBundle, isExternalSchemaRootActive } from '../../validation/schema-loader'; +import { + hasSchemaBundle, + isExternalSchemaRootActive, + withServedValidationVersion, +} from '../../validation/schema-loader'; import { ADCP_VERSION, LIBRARY_VERSION } from '../../version'; import { isPre31AdcpVersion } from '../../utils/adcp-version-config'; import { gte as semverGte, valid as validSemver } from 'semver'; @@ -536,6 +540,32 @@ function validateResponseSchema( validation: StoryboardValidation, ctx: ValidationContext, taskResult: TaskResult +): ValidationResult { + // Capabilities are negotiated before any pin applies: a 3.0 pin is served + // the agent's highest release in the major, echoed in `adcp_version`. + // Validate against that release (versioning spec, "Server echoes") when it + // is same-major, listed in the response's `supported_versions` and has a + // schema root, even when an external root is active for the pin. Otherwise + // keep the pin. Every other task stays on the pin. + const pinnedVersion = ctx.adcpVersion; + if (ctx.taskName === 'get_adcp_capabilities' && pinnedVersion !== undefined) { + return withServedValidationVersion(taskResult.data, pinnedVersion, validationVersion => + validateResponseSchemaForVersion( + validation, + validationVersion === pinnedVersion + ? ctx + : { ...ctx, adcpVersion: validationVersion, responseAdcpVersion: validationVersion }, + taskResult + ) + ); + } + return validateResponseSchemaForVersion(validation, ctx, taskResult); +} + +function validateResponseSchemaForVersion( + validation: StoryboardValidation, + ctx: ValidationContext, + taskResult: TaskResult ): ValidationResult { const taskName = ctx.taskName; const schemaIdentityVersion = ctx.adcpVersion ?? ctx.responseAdcpVersion ?? ADCP_VERSION; @@ -689,10 +719,21 @@ function validateResponseSchema( /** @internal Shared with the storyboard runner's Zod-rejection recovery path. */ export function isExternalResponseSchemaAuthoritative( - ctx: Pick + ctx: Pick & + Partial> ): boolean { const validationVersion = responseSchemaValidationVersion(ctx.adcpVersion, ctx.responseAdcpVersion); - return validationVersion !== undefined && isExternalSchemaRootActive(validationVersion); + if (validationVersion !== undefined && isExternalSchemaRootActive(validationVersion)) return true; + // A capabilities response validated against its echoed release is + // authoritative when that release's root is external (registered, or + // supplied through servedReleaseSchemaRoots), same as a pinned external root. + const pinnedVersion = ctx.adcpVersion; + if (ctx.taskName !== 'get_adcp_capabilities' || pinnedVersion === undefined || !ctx.taskResult) return false; + return withServedValidationVersion( + ctx.taskResult.data, + pinnedVersion, + version => version !== pinnedVersion && isExternalSchemaRootActive(version) + ); } function responseSchemaValidationVersion( diff --git a/src/lib/testing/types.ts b/src/lib/testing/types.ts index 0d6da32d1..0c1c38920 100644 --- a/src/lib/testing/types.ts +++ b/src/lib/testing/types.ts @@ -126,6 +126,45 @@ export interface TestOptions { * authoritative over the SDK's generated Zod snapshot for the run. */ schemaRoot?: string; + /** + * Schema roots for releases the agent may echo in its + * `get_adcp_capabilities` response, keyed by AdCP version (`'3.2'`, + * `'3.2.5'`, `'3.3.0-beta.1'`). Values are schema-data roots in the same + * shape as `schemaRoot`. + * + * A 3.0 pin sends only `adcp_major_version`, so an agent may serve and echo + * a newer release in the same major (versioning spec, "Server echoes"). + * The SDK validates that capabilities response against the echoed release + * when the echo is the pin's major, is listed in the response's + * `adcp.supported_versions`, and has a schema root: an entry here, an + * external root registered for it, or the SDK's bundled schemas. That + * applies to discovery's capability notices, `getAdcpCapabilities()` and + * storyboard `response_schema` checks on `get_adcp_capabilities`, including + * when `schemaRoot` is set for the pin. Otherwise validation stays on the + * pin. + * + * Entries are used for that capabilities validation only. Every other + * task's response, storyboard selection and grading stay on the pin. Each + * root is checked against its key once per process; a malformed map throws + * before the run starts. Honored by `comply()`, `runStoryboard()` and + * `runStoryboardStep()`; other callers can use `withServedReleaseSchemaRoots`. + * The echo is compared with the run's pin, so storyboard checks apply it + * only when the run has one (`adcpVersion`, or the storyboard or compliance + * bundle's `adcp_version`); `comply()` always does. + * + * @example + * ```ts + * await comply(agentUrl, { + * complianceDir: hosted30.complianceDir, + * schemaRoot: hosted30.schemaRoot, + * servedReleaseSchemaRoots: { + * '3.1.29': hosted31.schemaRoot, + * '3.2.5': hosted32.schemaRoot, + * }, + * }); + * ``` + */ + servedReleaseSchemaRoots?: Readonly>; /** * Whether strict AJV response-schema failures contribute to storyboard * grading. Defaults to `true`, matching the hosted compliance grader. diff --git a/src/lib/validation/index.ts b/src/lib/validation/index.ts index 9f15a7ff2..989f4c859 100644 --- a/src/lib/validation/index.ts +++ b/src/lib/validation/index.ts @@ -17,6 +17,7 @@ export { registerExternalSchemaRoot, unregisterExternalSchemaRoot, withExternalSchemaRoot, + withServedReleaseSchemaRoots, } from './schema-loader'; export type { Direction, ResponseVariant } from './schema-loader'; export { validateOutgoingRequest, validateIncomingResponse, resolveValidationModes } from './client-hooks'; diff --git a/src/lib/validation/schema-loader.ts b/src/lib/validation/schema-loader.ts index d0e76b89a..470cdf4ef 100644 --- a/src/lib/validation/schema-loader.ts +++ b/src/lib/validation/schema-loader.ts @@ -1280,70 +1280,230 @@ export function hasSchemaBundle(version: string): boolean { } } +/** + * A caller-supplied schema root for a release an agent may echo in a + * `get_adcp_capabilities` response. `aliases` is the validated alias set + * (computed once per process per `version`/`root` pair). + */ +interface ServedReleaseSchemaRoot { + readonly version: string; + readonly root: string; + readonly aliases: ReadonlySet; +} + +const servedReleaseSchemaRootScope = new AsyncLocalStorage(); +/** + * Validated alias sets for served-release roots, keyed by `version\0root`. + * Checking a root walks every schema file in it (about a second for a full + * release), so a hosted caller that passes the same roots on every run pays + * that once per process. Roots are treated as immutable artifacts, and + * neither this memo nor the compiled validators for a root are evicted, so + * pass stable root paths rather than a fresh extraction per run. + */ +const servedReleaseRootAliases = new Map>(); + +/** + * Run `fn` with schema roots for the releases an agent may echo in its + * `get_adcp_capabilities` response. + * + * Keys are AdCP versions in any form the SDK accepts for a pin (`'3.2'`, + * `'3.2.5'`, `'3.3.0-beta.1'`); values are schema-data roots in the same + * shape as `schemaRoot` (a directory holding `bundled/`, e.g. a hosted + * `dist/schemas/3.2.5`). Each root is checked against its key the same way + * `withExternalSchemaRoot` checks one, and a malformed map throws before `fn` + * runs. + * + * The roots are consulted only when validating a `get_adcp_capabilities` + * response against the release the agent echoed (see + * {@link resolveServedValidationVersion}): discovery's capability notices, + * `getAdcpCapabilities()` response validation and storyboard + * `response_schema` checks on that task. They are not activated for any other + * task, so every other response, storyboard selection and grading stay on the + * pin. A supplied root takes precedence over an external root registered for + * the same release and over the SDK's bundled schemas for it. + * + * Scoped to the current async execution chain; nested calls add to the + * enclosing scope, with the innermost entry for a release winning. + * `comply()` and `runStoryboard()` accept the same map as + * `servedReleaseSchemaRoots` and call this for you. + */ +export function withServedReleaseSchemaRoots(roots: Readonly> | undefined, fn: () => T): T { + if (roots === undefined) return fn(); + if (roots === null || typeof roots !== 'object' || Array.isArray(roots)) { + throw new TypeError('servedReleaseSchemaRoots must be an object mapping AdCP versions to schema roots'); + } + const entries: ServedReleaseSchemaRoot[] = []; + for (const [version, root] of Object.entries(roots)) { + resolveBundleKey(version); + if (typeof root !== 'string' || root.length === 0) { + throw new TypeError(`servedReleaseSchemaRoots["${version}"] must be a non-empty schema root path`); + } + const memoKey = `${version}\0${root}`; + let aliases = servedReleaseRootAliases.get(memoKey); + if (!aliases) { + if (!hasSchemaRootShape(root)) { + throw new Error(`Served-release AdCP schema root for version "${version}" not found or empty at ${root}`); + } + aliases = assertSchemaRootMatchesVersion(version, root); + servedReleaseRootAliases.set(memoKey, aliases); + } + const conflict = entries.find(entry => entry.root !== root && [...aliases].some(alias => entry.aliases.has(alias))); + if (conflict) { + throw new Error( + `servedReleaseSchemaRoots["${version}"] and servedReleaseSchemaRoots["${conflict.version}"] name the same ` + + `release with different schema roots (${root}, ${conflict.root})` + ); + } + entries.push({ version, root, aliases }); + } + if (entries.length === 0) return fn(); + const parent = servedReleaseSchemaRootScope.getStore() ?? []; + return servedReleaseSchemaRootScope.run([...entries, ...parent], fn); +} + +function findServedReleaseSchemaRoot(release: string): ServedReleaseSchemaRoot | undefined { + const entries = servedReleaseSchemaRootScope.getStore(); + if (!entries?.length) return undefined; + let key: string; + try { + key = resolveBundleKey(release); + } catch { + return undefined; + } + return entries.find(entry => entry.aliases.has(key) || entry.aliases.has(release)); +} + /** Major of an echoed release-precision `adcp_version` (`core/version-envelope.json` wire pattern). */ const SERVED_RELEASE_MAJOR = /^(\d+)\.\d+(?:-[a-zA-Z0-9.-]+)?$/; /** Major of a numeric pin: `'3.0'`, `'3.0.25'`, `'3.1-beta'`, `'3.1.0-beta.1'`. */ const NUMERIC_PIN_MAJOR = /^(\d+)\.\d+/; +/** Whether the response's own `adcp.supported_versions` lists `release`. */ +function listsSupportedRelease(response: object, release: string): boolean { + const adcp = (response as { adcp?: unknown }).adcp; + if (adcp === null || typeof adcp !== 'object') return false; + const supported = (adcp as { supported_versions?: unknown }).supported_versions; + if (!Array.isArray(supported)) return false; + return supported.some(entry => { + if (typeof entry !== 'string') return false; + if (entry === release) return true; + try { + return toReleasePrecisionWire(entry) === release; + } catch { + return false; + } + }); +} + +interface ServedValidationSelection { + /** Version to validate against: the echoed release, or the pin. */ + version: string; + /** Caller-supplied root for the echoed release, when one applies. */ + served?: ServedReleaseSchemaRoot; +} + +function selectServedValidationVersion(response: unknown, pinnedVersion: string): ServedValidationSelection { + const pin: ServedValidationSelection = { version: pinnedVersion }; + const pinMajor = NUMERIC_PIN_MAJOR.exec(pinnedVersion)?.[1]; + // Legacy aliases ('v2.5', 'v3') have their own adapter paths; leave them alone. + if (pinMajor === undefined) return pin; + if (response === null || typeof response !== 'object') return pin; + + const echoed = (response as { adcp_version?: unknown }).adcp_version; + if (typeof echoed !== 'string') return pin; + const echoMajor = SERVED_RELEASE_MAJOR.exec(echoed)?.[1]; + if (echoMajor === undefined || echoMajor !== pinMajor) return pin; + + let pinnedRelease: string; + try { + pinnedRelease = toReleasePrecisionWire(pinnedVersion); + } catch { + return pin; + } + // The pinned release itself: keep the exact pin so a patch-precise pin and + // any external root bound to it still apply. + if (echoed === pinnedRelease) return pin; + // An agent may only serve a release it advertises. + if (!listsSupportedRelease(response, echoed)) return pin; + + const served = findServedReleaseSchemaRoot(echoed); + if (served) return { version: echoed, served }; + // Covers an external root registered (or scoped) for the echo and the + // SDK's bundled schemas. + return hasSchemaBundle(echoed) ? { version: echoed } : pin; +} + /** - * Select the AdCP release a seller response is validated against. + * Select the AdCP release a `get_adcp_capabilities` response is validated + * against. * * The versioning spec (`docs/reference/versioning.mdx`, "Server echoes") says * the seller echoes the release it actually served in `adcp_version`, and * buyers SHOULD validate against that release's schema, not their pin; with - * no echo, against the pin. This matters most for `get_adcp_capabilities` - * under a 3.0 pin: the 3.0 bundle has no `adcp_version` request field, so the - * SDK sends only `adcp_major_version: 3` and the seller serves (and echoes) - * its highest 3.x release. + * no echo, against the pin. This matters most under a 3.0 pin: the 3.0 + * bundle has no `adcp_version` request field, so the SDK sends only + * `adcp_major_version: 3` and the seller serves (and echoes) its highest 3.x + * release. * - * Returns the version to validate `response` against: the echoed - * `adcp_version` when it is a usable release, otherwise `pinnedVersion`. + * Returns the echoed `adcp_version` when all of these hold, otherwise + * `pinnedVersion`: + * - it is a release-precision string in the same major as the pin; + * - the response's own `adcp.supported_versions` lists it; + * - a schema root for it is available: one supplied through + * {@link withServedReleaseSchemaRoots} (`servedReleaseSchemaRoots` on + * `comply()` / `runStoryboard()`), an external root registered or scoped + * for it, or the SDK's bundled schemas. * - * The echo is used only when it is a release-precision string in the same - * major as the pin and the SDK has a schema bundle for it. Validation stays - * strict either way: an echo from another major, a malformed echo, or one the - * SDK cannot validate falls back to the pin rather than skipping validation. - * When the echo names the pinned release itself, the exact pin is returned so - * a patch-precise pin (and any external schema root bound to it) still - * applies. `supported_versions` is deliberately not consulted: with no echo - * the spec says to validate against the pin. + * This holds whether or not an external schema root is active for the pin. + * Hosted compliance always runs with one, and the capabilities response is + * negotiated before any pin applies, so the pin's root must not override the + * release the agent actually served. * - * Two deliberate consequences: - * - An external schema root the caller bound to the pin (`schemaRoot`, - * `withExternalSchemaRoot`) wins over the echo, matching the storyboard - * runner's `response_schema` precedence: the caller chose that protocol - * source explicitly. - * - An echo older than the pin (pin 3.2, served 3.0) is validated against the - * older release. That is the spec's downshift rule, not a loosening bug. + * Validation stays strict either way: an echo from another major, a malformed + * or unlisted echo, or one with no schema available falls back to the pin + * rather than skipping validation. When the echo names the pinned release + * itself, the exact pin is returned so a patch-precise pin (and any external + * schema root bound to it) still applies. An echo older than the pin (pin + * 3.2, served 3.0) is validated against the older release; that is the spec's + * downshift rule, not a loosening bug. + * + * Callers apply this to `get_adcp_capabilities` only. Use + * {@link withServedValidationVersion} to also activate a supplied root. * * @internal — not part of the public API surface; may change without a major bump. */ export function resolveServedValidationVersion(response: unknown, pinnedVersion: string): string { - const pinMajor = NUMERIC_PIN_MAJOR.exec(pinnedVersion)?.[1]; - // Legacy aliases ('v2.5', 'v3') have their own adapter paths; leave them alone. - if (pinMajor === undefined) return pinnedVersion; try { - if (isExternalSchemaRootActive(pinnedVersion)) return pinnedVersion; + return selectServedValidationVersion(response, pinnedVersion).version; } catch { return pinnedVersion; } +} - const echoed = - response !== null && typeof response === 'object' - ? (response as { adcp_version?: unknown }).adcp_version - : undefined; - if (typeof echoed !== 'string') return pinnedVersion; - const echoMajor = SERVED_RELEASE_MAJOR.exec(echoed)?.[1]; - if (echoMajor === undefined || echoMajor !== pinMajor) return pinnedVersion; - - let pinnedRelease: string; +/** + * Run `fn` with the release {@link resolveServedValidationVersion} selects. + * When that release has a root supplied through + * {@link withServedReleaseSchemaRoots}, the root is active for the duration + * of `fn` only, so it never leaks into validation of any other response. + * + * @internal — not part of the public API surface; may change without a major bump. + */ +export function withServedValidationVersion( + response: unknown, + pinnedVersion: string, + fn: (version: string) => T +): T { + let selection: ServedValidationSelection; try { - pinnedRelease = toReleasePrecisionWire(pinnedVersion); + selection = selectServedValidationVersion(response, pinnedVersion); } catch { - return pinnedVersion; + selection = { version: pinnedVersion }; } - if (echoed === pinnedRelease) return pinnedVersion; - return hasSchemaBundle(echoed) ? echoed : pinnedVersion; + const { version, served } = selection; + if (!served) return fn(version); + const next = new Map(scopedExternalSchemaRoots.getStore()); + for (const alias of served.aliases) next.set(alias, served.root); + return scopedExternalSchemaRoots.run(next, () => fn(version)); } const mcpToolSchemaCache = new Map> | null>(); diff --git a/test/lib/capabilities-echoed-release-hosted-validation.test.js b/test/lib/capabilities-echoed-release-hosted-validation.test.js new file mode 100644 index 000000000..c64a90272 --- /dev/null +++ b/test/lib/capabilities-echoed-release-hosted-validation.test.js @@ -0,0 +1,501 @@ +/** + * `get_adcp_capabilities` is validated against the release the agent echoed + * even when an external schema root is active for the pin. + * + * Hosted compliance always runs with an external schema root for the pinned + * release (`comply()` derives it from the compliance folder). A 3.0 pin sends + * only `adcp_major_version: 3`, so a conformant agent serves and echoes its + * highest 3.x release (versioning spec, "Server echoes"). Before the fix the + * pin's external root won over the echo, so every storyboard step's + * `response_schema` check on a 3.2 capabilities response, and the discovery + * notices, validated against 3.0 and failed on fields 3.0 never had. + * + * The echo is used only when it is the pin's major, is listed in the + * response's own `adcp.supported_versions`, and has a schema root (one + * supplied through `servedReleaseSchemaRoots`, an external root registered + * for it, or the SDK's bundled schemas). Otherwise validation stays strictly + * on the pin, and no other task's validation changes. + */ + +const { describe, test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); + +process.env.ADCP_ALLOW_INTERNAL_PROBES = '1'; + +const { ADCPMultiAgentClient } = require('../../dist/lib/index.js'); +const { testCapabilityDiscovery } = require('../../dist/lib/testing/scenarios/capabilities.js'); +const { runValidations } = require('../../dist/lib/testing/storyboard/validations.js'); +const { runStoryboard } = require('../../dist/lib/testing/storyboard/runner.js'); +const { closeMCPConnections } = require('../../dist/lib/protocols/mcp.js'); +const { + isExternalSchemaRootActive, + resolveServedValidationVersion, + withExternalSchemaRoot, + withServedReleaseSchemaRoots, +} = require('../../dist/lib/validation/schema-loader.js'); +const testing = require('../../dist/lib/testing/index.js'); + +const SCHEMAS_DATA = path.join(__dirname, '../../dist/lib/schemas-data'); +/** Stands in for hosted compliance's external root for a 3.0 target. */ +const PINNED_30_ROOT = path.join(SCHEMAS_DATA, '3.0'); +const MARKER = 'x_hosted_root_marker'; +const CAPABILITIES_FILE = path.join('protocol', 'get-adcp-capabilities-response.json'); +const PRODUCTS_FILE = path.join('media-buy', 'get-products-response.json'); + +/** A conformant AdCP 3.2 capabilities response, invalid against 3.0. */ +function capabilities32(overrides = {}) { + return { + status: 'completed', + adcp_version: '3.2', + adcp: { + major_versions: [3], + supported_versions: ['3.0', '3.1', '3.2'], + idempotency: { supported: true, replay_ttl_seconds: 86400 }, + }, + supported_protocols: ['media_buy'], + account: { require_operator_auth: false, supported_billing: ['operator'] }, + // 3.2 widened `features` entries: 3.0 requires booleans here. + media_buy: { features: { bidding_policy: { media_buy: { fixed: { modes: ['bid_amount'] } } } } }, + // 3.0 closes `scenarios` to six values; 3.1+ opens the list. + compliance_testing: { scenarios: ['force_creative_status', 'force_create_media_buy_arm', 'seed_product'] }, + ...overrides, + }; +} + +function withSupportedVersions(payload, supported_versions) { + return { ...payload, adcp: { ...payload.adcp, supported_versions } }; +} + +/** The 3.0 pin rejects capabilities32() at these pointers. */ +const PIN_30_ISSUE = /must be boolean|must be one of/; + +/** + * A hosted-style 3.2.5 schema root: the SDK's bundled 3.2 capabilities and + * get_products response schemas, each requiring an extra marker field, so a + * test can tell this root apart from the bundled one. + */ +let hosted32Root; +/** A root for 3.9, a release the SDK does not bundle (3.2 shapes, 3.9 ids). */ +let unbundled39Root; +let tmpDir; + +before(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'adcp-echoed-release-')); + hosted32Root = path.join(tmpDir, '3.2.5'); + for (const file of [CAPABILITIES_FILE, PRODUCTS_FILE]) { + const schema = JSON.parse(fs.readFileSync(path.join(SCHEMAS_DATA, '3.2', 'bundled', file), 'utf8')); + schema.required = [...(schema.required ?? []), MARKER]; + const target = path.join(hosted32Root, 'bundled', file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, JSON.stringify(schema)); + } + unbundled39Root = path.join(tmpDir, '3.9.0'); + const capabilities = fs + .readFileSync(path.join(SCHEMAS_DATA, '3.2', 'bundled', CAPABILITIES_FILE), 'utf8') + .replaceAll('/schemas/3.2.5/', '/schemas/3.9.0/'); + const target39 = path.join(unbundled39Root, 'bundled', CAPABILITIES_FILE); + fs.mkdirSync(path.dirname(target39), { recursive: true }); + fs.writeFileSync(target39, capabilities); +}); + +after(async () => { + await closeMCPConnections(); + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +const underPinnedRoot = fn => withExternalSchemaRoot('3.0', PINNED_30_ROOT, fn); + +function responseSchemaCheck(taskName, data, extra = {}) { + const [result] = runValidations([{ check: 'response_schema', description: 'response conforms' }], { + taskName, + adcpVersion: '3.0', + responseAdcpVersion: '3.0', + strictResponseSchemaValidation: true, + taskResult: { success: true, data }, + agentUrl: 'https://agent.example/mcp', + contributions: new Set(), + ...extra, + }); + return result; +} + +describe('resolveServedValidationVersion with an external root active for the pin', () => { + test('uses a same-major echo listed in supported_versions and bundled by the SDK', () => { + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(capabilities32(), '3.0')), + '3.2' + ); + }); + + test('keeps the pin when the echo is a different major', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '4.0' }), ['3.0', '4.0']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0' + ); + }); + + test('keeps the pin when the echo is not listed in supported_versions', () => { + const unlisted = withSupportedVersions(capabilities32(), ['3.0', '3.1']); + const missing = capabilities32({ adcp: { major_versions: [3], idempotency: { supported: false } } }); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(unlisted, '3.0')), + '3.0' + ); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(missing, '3.0')), + '3.0' + ); + // Same rule without any external root. + assert.equal(resolveServedValidationVersion(unlisted, '3.0'), '3.0'); + }); + + test('keeps the pin when no schema is available for the echo', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.99' }), ['3.0', '3.99']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0' + ); + }); + + test('an unbundled echo becomes usable once a served or external root is supplied for it', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.9' }), ['3.0', '3.2', '3.9']); + assert.equal( + underPinnedRoot(() => resolveServedValidationVersion(payload, '3.0')), + '3.0', + 'no root: pin' + ); + const viaServed = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.9.0': unbundled39Root }, () => resolveServedValidationVersion(payload, '3.0')) + ); + assert.equal(viaServed, '3.9'); + const viaExternal = underPinnedRoot(() => + withExternalSchemaRoot('3.9.0', unbundled39Root, () => resolveServedValidationVersion(payload, '3.0')) + ); + assert.equal(viaExternal, '3.9'); + }); +}); + +describe('withServedReleaseSchemaRoots', () => { + test('is exported from @adcp/sdk/testing', () => { + assert.equal(typeof testing.withServedReleaseSchemaRoots, 'function'); + }); + + test('rejects malformed input up front', () => { + assert.throws(() => withServedReleaseSchemaRoots(['3.2'], () => 1), /servedReleaseSchemaRoots/); + assert.throws(() => withServedReleaseSchemaRoots({ '../3.2': hosted32Root }, () => 1), /not a recognized version/); + assert.throws(() => withServedReleaseSchemaRoots({ '3.2.5': '' }, () => 1), /servedReleaseSchemaRoots/); + assert.throws( + () => withServedReleaseSchemaRoots({ '3.2.5': path.join(tmpDir, 'missing') }, () => 1), + /not found or empty/ + ); + // A root whose schema ids belong to another release. + assert.throws(() => withServedReleaseSchemaRoots({ 3.1: hosted32Root }, () => 1), /does not match/); + // Two keys for the same release with different roots. + assert.throws( + () => withServedReleaseSchemaRoots({ '3.2.5': hosted32Root, 3.2: path.join(SCHEMAS_DATA, '3.2') }, () => 1), + /same release with different schema roots/ + ); + }); + + test('is a no-op without entries', () => { + assert.equal( + withServedReleaseSchemaRoots(undefined, () => 7), + 7 + ); + assert.equal( + withServedReleaseSchemaRoots({}, () => 8), + 8 + ); + }); +}); + +describe('storyboard response_schema check on get_adcp_capabilities under an external pin root', () => { + test('passes a conformant response for the echoed same-major release', () => { + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', capabilities32())); + assert.equal(result.passed, true, result.error); + }); + + test('still fails a real schema error under the echoed release', () => { + // minProperties: 1 on the 3.2 bidding_policy object. + const payload = capabilities32({ media_buy: { features: { bidding_policy: {} } } }); + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(result.passed, false); + assert.match(result.error, /bidding_policy/); + assert.doesNotMatch(result.error, PIN_30_ISSUE, 'validated against 3.2, not the 3.0 pin'); + }); + + for (const [label, payload] of [ + ['an echo of a different major', withSupportedVersions(capabilities32({ adcp_version: '4.0' }), ['3.0', '4.0'])], + ['an echo not in supported_versions', withSupportedVersions(capabilities32(), ['3.0', '3.1'])], + [ + 'an echo with no schema available', + withSupportedVersions(capabilities32({ adcp_version: '3.99' }), ['3.0', '3.99']), + ], + ['no echo', (({ adcp_version: _omitted, ...rest }) => rest)(capabilities32())], + ]) { + test(`falls back to the pin, strictly, for ${label}`, () => { + const result = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(result.passed, false, 'the 3.0 pin rejects 3.2 shapes'); + assert.match(result.error, PIN_30_ISSUE); + }); + } + + test('uses the root supplied for the echoed release', () => { + const missingMarker = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + responseSchemaCheck('get_adcp_capabilities', capabilities32()) + ) + ); + assert.equal(missingMarker.passed, false); + assert.match(missingMarker.error, new RegExp(MARKER)); + + const withMarker = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + responseSchemaCheck('get_adcp_capabilities', capabilities32({ [MARKER]: true })) + ) + ); + assert.equal(withMarker.passed, true, withMarker.error); + }); + + test('validates an unbundled echo against the root supplied for it', () => { + const payload = withSupportedVersions(capabilities32({ adcp_version: '3.9' }), ['3.0', '3.9']); + const unsupplied = underPinnedRoot(() => responseSchemaCheck('get_adcp_capabilities', payload)); + assert.equal(unsupplied.passed, false, 'no 3.9 schema: strict pin'); + assert.match(unsupplied.error, PIN_30_ISSUE); + + const supplied = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.9.0': unbundled39Root }, () => + responseSchemaCheck('get_adcp_capabilities', payload) + ) + ); + assert.equal(supplied.passed, true, supplied.error); + }); + + test('does not change other tasks: they validate against the pin and ignore served roots', () => { + // A get_products response that echoes 3.2 and lists it is still graded + // against the pinned 3.0 root, never the served 3.2 root. + const products = { + adcp_version: '3.2', + adcp: { supported_versions: ['3.0', '3.2'] }, + products: [], + }; + const baseline = underPinnedRoot(() => responseSchemaCheck('get_products', { products: [] })); + const echoed = underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => responseSchemaCheck('get_products', products)) + ); + assert.equal(echoed.passed, baseline.passed, echoed.error); + assert.equal(echoed.schema_id, baseline.schema_id); + assert.doesNotMatch(echoed.error ?? '', new RegExp(MARKER)); + }); + + test('served roots never become active for other tasks, even on the served release', () => { + // Pin 3.2 with a supplied 3.2.5 root: a get_products response is still + // validated against the SDK's bundled 3.2, which has no marker field. + const extra = { + adcpVersion: '3.2', + responseAdcpVersion: '3.2', + responseSchemaRef: 'media-buy/get-products-response.json', + }; + const baseline = responseSchemaCheck('get_products', { products: [] }, extra); + const scoped = withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => { + assert.equal(isExternalSchemaRootActive('3.2'), false, 'scope does not register the root'); + return responseSchemaCheck('get_products', { products: [] }, extra); + }); + assert.equal(scoped.passed, baseline.passed, scoped.error); + assert.ok(scoped.schema_id, 'schema identity resolved'); + assert.equal(scoped.schema_id, baseline.schema_id); + assert.doesNotMatch(scoped.error ?? '', new RegExp(MARKER)); + }); +}); + +// ── Live seller: discovery notices, TaskExecutor and runStoryboard ────────── + +async function startSeller({ payload }) { + const { createMcpHandler, McpServer } = require('@modelcontextprotocol/server'); + const { toNodeHandler } = require('@modelcontextprotocol/node'); + const { z } = require('zod'); + const handler = createMcpHandler(() => { + const server = new McpServer({ name: 'echoed-release-seller', version: '1.0.0' }); + server.registerTool( + 'get_adcp_capabilities', + { description: 'AdCP capabilities', inputSchema: z.object({}).passthrough() }, + async () => ({ content: [{ type: 'text', text: JSON.stringify(payload) }], structuredContent: payload }) + ); + for (const name of ['get_products', 'create_media_buy', 'comply_test_controller']) { + server.registerTool(name, { description: name }, async () => ({ + content: [{ type: 'text', text: '{}' }], + structuredContent: {}, + })); + } + return server; + }); + const serveMcp = toNodeHandler(handler); + const server = http.createServer(async (req, res) => { + if (req.url !== '/mcp' && req.url !== '/mcp/') { + res.writeHead(404).end('not found'); + return; + } + await serveMcp(req, res); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + return { + url: `http://127.0.0.1:${server.address().port}/mcp`, + async close() { + await handler.close(); + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + }, + }; +} + +async function withSeller(t, payload) { + await closeMCPConnections(); + const seller = await startSeller({ payload }); + t.after(async () => { + await closeMCPConnections(); + await seller.close(); + }); + return seller; +} + +function capabilitiesStoryboard() { + return { + id: 'echoed_release_capabilities', + version: '1.0.0', + title: 'Echoed-release capabilities', + category: 'test', + summary: 'A single get_adcp_capabilities step with a response_schema check.', + narrative: '', + agent: { interaction_model: 'sync', capabilities: [] }, + caller: { role: 'buyer_agent' }, + phases: [ + { + id: 'capabilities', + title: 'Capabilities', + steps: [ + { + id: 'discover', + title: 'Discover capabilities', + task: 'get_adcp_capabilities', + response_schema_ref: 'protocol/get-adcp-capabilities-response.json', + validations: [{ check: 'response_schema', description: 'response conforms' }], + }, + ], + }, + ], + }; +} + +describe('live seller serving 3.2 to a 3.0 pin with an external pin root', () => { + test('comply() with a 3.0 compliance cache and pin root grades the echoed 3.2 capabilities', async t => { + // Hosted compliance shape: comply() opens the external 3.0 root itself. + const seller = await withSeller(t, capabilities32()); + const run = extra => + testing.comply(seller.url, { + version: '3.0.25', + schemaRoot: PINNED_30_ROOT, + storyboards: ['capability_discovery'], + protocol: 'mcp', + allow_http: true, + ...extra, + }); + const schemaFailures = result => (result.failures ?? []).filter(f => f.validation?.check === 'response_schema'); + const capabilityNotices = result => + (result.notices ?? []).filter(n => n.code === 'capabilities_response_schema_invalid'); + + const result = await run({}); + assert.deepEqual(schemaFailures(result), [], JSON.stringify(schemaFailures(result), null, 2)); + assert.deepEqual(capabilityNotices(result), []); + + await closeMCPConnections(); + const served = await run({ servedReleaseSchemaRoots: { '3.2.5': hosted32Root } }); + const failures = schemaFailures(served); + assert.ok(failures.length > 0, 'the supplied 3.2.5 root grades the capabilities steps'); + assert.ok( + failures.every(f => JSON.stringify(f).includes(MARKER)), + JSON.stringify(failures, null, 2) + ); + assert.ok( + capabilityNotices(served).some(n => JSON.stringify(n).includes(MARKER)), + JSON.stringify(served.notices, null, 2) + ); + + await assert.rejects( + run({ servedReleaseSchemaRoots: { '3.2.5': path.join(tmpDir, 'missing') } }), + /not found or empty/ + ); + }); + + test('discovery reports no capability schema issues for a conformant 3.2 response', async t => { + const seller = await withSeller(t, capabilities32()); + const discovery = await underPinnedRoot(() => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ); + assert.equal(discovery.profile?.capabilities_schema_issues, undefined); + }); + + test('discovery validates against the supplied root for the echoed release', async t => { + const seller = await withSeller(t, capabilities32()); + const discovery = await underPinnedRoot(() => + withServedReleaseSchemaRoots({ '3.2.5': hosted32Root }, () => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ) + ); + const issues = discovery.profile?.capabilities_schema_issues ?? []; + assert.ok( + issues.some(issue => issue.pointer === `/${MARKER}` || issue.message.includes(MARKER)), + JSON.stringify(issues) + ); + assert.ok( + issues.every(issue => !PIN_30_ISSUE.test(issue.message)), + JSON.stringify(issues) + ); + }); + + test('discovery keeps strict pin validation for an echo not in supported_versions', async t => { + const seller = await withSeller(t, withSupportedVersions(capabilities32(), ['3.0', '3.1'])); + const discovery = await underPinnedRoot(() => + testCapabilityDiscovery(seller.url, { protocol: 'mcp', adcpVersion: '3.0' }) + ); + const issues = discovery.profile?.capabilities_schema_issues ?? []; + assert.ok( + issues.some(issue => PIN_30_ISSUE.test(issue.message)), + JSON.stringify(issues) + ); + }); + + test('TaskExecutor accepts getAdcpCapabilities in strict mode', async t => { + const seller = await withSeller(t, capabilities32()); + const client = new ADCPMultiAgentClient( + [{ id: 'seller', name: 'Seller', agent_uri: seller.url, protocol: 'mcp' }], + { adcpVersion: '3.0', validation: { responses: 'strict', logSchemaViolations: false } } + ); + const result = await underPinnedRoot(() => client.agent('seller').getAdcpCapabilities({})); + assert.equal(result.success, true, result.error); + }); + + test('runStoryboard passes the capabilities step and honors servedReleaseSchemaRoots', async t => { + const seller = await withSeller(t, capabilities32()); + const options = { protocol: 'mcp', adcpVersion: '3.0', schemaRoot: PINNED_30_ROOT }; + + const passed = await runStoryboard(seller.url, capabilitiesStoryboard(), options); + const step = passed.phases[0].steps[0]; + const check = step.validations.find(v => v.check === 'response_schema'); + assert.equal(check?.passed, true, JSON.stringify(step, null, 2)); + + await closeMCPConnections(); + const served = await runStoryboard(seller.url, capabilitiesStoryboard(), { + ...options, + servedReleaseSchemaRoots: { '3.2.5': hosted32Root }, + }); + const servedStep = served.phases[0].steps[0]; + const servedCheck = servedStep.validations.find(v => v.check === 'response_schema'); + assert.equal(servedCheck?.passed, false, JSON.stringify(servedStep, null, 2)); + assert.match(servedCheck.error, new RegExp(MARKER)); + }); +}); diff --git a/test/lib/capabilities-served-release-validation.test.js b/test/lib/capabilities-served-release-validation.test.js index 77444c780..1a4900dbd 100644 --- a/test/lib/capabilities-served-release-validation.test.js +++ b/test/lib/capabilities-served-release-validation.test.js @@ -106,55 +106,63 @@ async function withSeller(t, options) { const stepNamed = (steps, name) => steps.find(step => step.step === name); const failedSteps = steps => steps.filter(step => !step.passed && !step.skipped); +/** An echo of `release` that the response also lists in `supported_versions`. */ +const echo = (release, supported = ['3.0', '3.1', '3.2', release]) => ({ + adcp_version: release, + adcp: { supported_versions: [...new Set(supported)] }, +}); + describe('resolveServedValidationVersion', () => { - test('uses a bundled same-major echo instead of the pin', () => { - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.2'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.1' }, '3.0.25'), '3.1'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.0' }, '3.2'), '3.0'); + test('uses a bundled same-major echo listed in supported_versions instead of the pin', () => { + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.0'), '3.2'); + assert.equal(resolveServedValidationVersion(echo('3.1'), '3.0.25'), '3.1'); + assert.equal(resolveServedValidationVersion(echo('3.0'), '3.2'), '3.0'); }); test('keeps the exact pin when the echo is the pinned release', () => { // Preserves a patch-precise pin (and any external schema root bound to it). - assert.equal(resolveServedValidationVersion({ adcp_version: '3.0' }, '3.0.25'), '3.0.25'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.2'), '3.2'); + assert.equal(resolveServedValidationVersion(echo('3.0'), '3.0.25'), '3.0.25'); + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.2'), '3.2'); }); test('falls back to the pin when there is no usable echo', () => { assert.equal(resolveServedValidationVersion({}, '3.0'), '3.0', 'no echo'); assert.equal(resolveServedValidationVersion(undefined, '3.0'), '3.0', 'no payload'); - assert.equal(resolveServedValidationVersion({ adcp_version: 3.2 }, '3.0'), '3.0', 'non-string echo'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2.4' }, '3.0'), '3.0', 'not release precision'); - assert.equal(resolveServedValidationVersion({ adcp_version: '../3.2' }, '3.0'), '3.0', 'garbage'); - assert.equal(resolveServedValidationVersion({ adcp_version: '4.0' }, '3.0'), '3.0', 'different major'); - assert.equal(resolveServedValidationVersion({ adcp_version: '2.5' }, '3.0'), '3.0', 'older major'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.99' }, '3.0'), '3.0', 'no bundled schema'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, 'v2.5'), 'v2.5', 'legacy v2 pin'); + assert.equal(resolveServedValidationVersion({ ...echo('3.2'), adcp_version: 3.2 }, '3.0'), '3.0', 'non-string'); + assert.equal(resolveServedValidationVersion(echo('3.2.4'), '3.0'), '3.0', 'not release precision'); + assert.equal(resolveServedValidationVersion(echo('../3.2'), '3.0'), '3.0', 'garbage'); + assert.equal(resolveServedValidationVersion(echo('4.0'), '3.0'), '3.0', 'different major'); + assert.equal(resolveServedValidationVersion(echo('2.5'), '3.0'), '3.0', 'older major'); + assert.equal(resolveServedValidationVersion(echo('3.99'), '3.0'), '3.0', 'no bundled schema'); + assert.equal(resolveServedValidationVersion(echo('3.2'), 'v2.5'), 'v2.5', 'legacy v2 pin'); }); - test('does not consult supported_versions', () => { - // Spec: with no echo, validate against the pin. + test('requires the echo to be listed in supported_versions', () => { + assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.0', 'no supported_versions'); + assert.equal(resolveServedValidationVersion(echo('3.2', ['3.0', '3.1']), '3.0'), '3.0', 'not listed'); + assert.equal( + resolveServedValidationVersion({ adcp_version: '3.2', adcp: { supported_versions: '3.2' } }, '3.0'), + '3.0', + 'malformed supported_versions' + ); + // supported_versions alone never moves validation off the pin. assert.equal(resolveServedValidationVersion({ adcp: { supported_versions: ['3.0', '3.2'] } }, '3.0'), '3.0'); }); test('handles prerelease and legacy pins', () => { - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.1-beta'), '3.2', 'stable echo, beta pin'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.1-beta.1' }, '3.1.0-beta.1'), '3.1.0-beta.1'); - assert.equal( - resolveServedValidationVersion({ adcp_version: '3.4-rc.1' }, '3.0'), - '3.0', - 'unbundled prerelease echo' - ); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, 'v3'), 'v3', 'legacy alias pin'); + assert.equal(resolveServedValidationVersion(echo('3.2'), '3.1-beta'), '3.2', 'stable echo, beta pin'); + assert.equal(resolveServedValidationVersion(echo('3.1-beta.1'), '3.1.0-beta.1'), '3.1.0-beta.1'); + assert.equal(resolveServedValidationVersion(echo('3.4-rc.1'), '3.0'), '3.0', 'unbundled prerelease echo'); + assert.equal(resolveServedValidationVersion(echo('3.2'), 'v3'), 'v3', 'legacy alias pin'); }); - test('an external schema root bound to the pin wins over the echo', () => { - // Same precedence as the storyboard runner's response_schema check. + test('an external schema root bound to the pin does not override a usable echo', () => { + // Hosted compliance always runs with one (adcp-client#3320). const root = path.join(__dirname, '../../dist/lib/schemas-data/3.0'); - const selected = withExternalSchemaRoot('3.0', root, () => - resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0') - ); - assert.equal(selected, '3.0'); - assert.equal(resolveServedValidationVersion({ adcp_version: '3.2' }, '3.0'), '3.2', 'scope ended'); + const selected = withExternalSchemaRoot('3.0', root, () => resolveServedValidationVersion(echo('3.2'), '3.0')); + assert.equal(selected, '3.2'); + const pinnedEcho = withExternalSchemaRoot('3.0', root, () => resolveServedValidationVersion(echo('3.0'), '3.0')); + assert.equal(pinnedEcho, '3.0', 'an echo of the pinned release keeps the pin and its root'); }); });