diff --git a/scripts/external-smoke/README.md b/scripts/external-smoke/README.md index ad5d4cad8cd..1f2a1304640 100644 --- a/scripts/external-smoke/README.md +++ b/scripts/external-smoke/README.md @@ -10,9 +10,25 @@ Each authenticated/anonymous probe makes at most two discovery attempts, each with a fresh AbortSignal and fetch closure, a 10s total budget, and explicit 4s SDK transport timeout. Only categorized transient transport failures retry; schema, assertion, auth and unknown errors do not. -Authenticated discovery runs once per attempt and its profile is reused for -local storyboard resolution. Anonymous success meets the same checks, or -failed discovery must supply concrete authentication error evidence. +Authenticated discovery runs once per negotiated release per attempt, and its +profile is reused for local storyboard resolution. Anonymous success meets the +same checks, or failed discovery must supply concrete authentication error +evidence (including the SDK's OAuth challenge, "requires OAuth authorization"). + +The probe pins the default hosted line (3.0). A 3.0 pin cannot carry +`adcp_version`, so the SDK sends only `adcp_major_version: 3`, and the agent +may serve and echo its highest 3.x release. `@adcp/sdk` validates the response +against the 3.0 pin regardless. The versioning spec says buyers validate +against the echoed release. When the echo differs from the pin, the smoke check +therefore requires the echo to be listed in `adcp.supported_versions`, +discovers again pinned to that release, validates strictly against it, and +resolves storyboards from that release's hosted bundle. An echo with no hosted +bundle, a different major, or a different release on the pinned probe fails. + +SDK 15.8.1 also keys `comply_test_controller` as protocol `compliance` in its +tool cross-check, while it parses the `compliance_testing` block as +`compliance_testing`. The check waives only that exact cross-check message, +and only when the response carries the `compliance_testing` block. SDK 14.0.0-rc.35 does not pass the signal to its second capability call. The local `trustedFetchFn` composes the attempt signal with the transport signal diff --git a/scripts/external-smoke/check.ts b/scripts/external-smoke/check.ts index 70d67380233..a807df44f6d 100644 --- a/scripts/external-smoke/check.ts +++ b/scripts/external-smoke/check.ts @@ -23,13 +23,16 @@ export class SmokeFailure extends Error { // SDK step errors are strings. Keep this allowlist narrow; unknown errors fail // once. In particular, a generic mention of "auth" or "network" proves nothing. +// `requires OAuth authorization` is the message of the SDK's +// NeedsAuthorizationError (a 401 Bearer challenge), which reaches us only as a +// step string. function failureKind(error: unknown): FailureKind { if (error instanceof SmokeFailure) return error.kind; const value = error as { name?: string; code?: string; status?: number; message?: string; cause?: unknown } | null; const evidence = typeof error === 'string' ? error : [value?.name, value?.code, value?.status, value?.message].join(' '); if (/\b(?:AssertionError|ZodError|SchemaValidationError)\b|schema|validation|unsupported version/i.test(evidence)) return 'semantic'; - if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token/i.test(evidence)) return 'auth'; + if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token|\brequires OAuth authorization\b/i.test(evidence)) return 'auth'; if ([408, 429, 502, 503, 504].includes(value?.status ?? 0) || /\b(?:ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET|TimeoutError|SmokeTransientTransportError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:408|429|502|503|504)\b|request timed out|request timeout|version negotiation probe timed out/i.test(evidence)) return 'transient'; return value?.cause ? failureKind(value.cause) : 'semantic'; } @@ -85,10 +88,29 @@ export async function boundedDiscovery( } } +// @adcp/sdk 15.8.1 (and 15.9.1) parses a `compliance_testing` capability +// block as protocol `compliance_testing`, but its tool cross-check keys +// comply_test_controller under `compliance`. Every agent that correctly +// declares the block therefore fails the step with exactly this message. +// Waive only that exact message, and only when the agent's response carries +// the block (the SDK also adds the protocol when the block is missing). Any +// other cross-check issue still fails. +const SDK_COMPLIANCE_CROSS_CHECK_STEP = 'Cross-validate protocols and tools'; +const SDK_COMPLIANCE_CROSS_CHECK_ERROR = 'Tools suggest unreported protocols: compliance (has: [comply_test_controller])'; + +function isSdkComplianceCrossCheckFalsePositive(step: DiscoveryResult['steps'][number], result: DiscoveryResult): boolean { + return step.step === SDK_COMPLIANCE_CROSS_CHECK_STEP + && step.error === SDK_COMPLIANCE_CROSS_CHECK_ERROR + && result.profile?.supported_protocols?.includes('compliance_testing') === true + && typeof (result.profile.raw_capabilities as { compliance_testing?: unknown } | undefined)?.compliance_testing === 'object'; +} + function assertDiscovery(result: DiscoveryResult): void { const schemaIssues = result.profile?.capabilities_schema_issues; if (schemaIssues?.length) throw new SmokeFailure('semantic', `Capability schema errors: ${JSON.stringify(schemaIssues)}`); - const failures = result.steps.filter(step => !step.passed && !step.skipped).map(step => ({ + const failed = result.steps.filter(step => !step.passed && !step.skipped + && !isSdkComplianceCrossCheckFalsePositive(step, result)); + const failures = failed.map(step => ({ kind: step.task === 'getAgentInfo' || step.task === 'get_adcp_capabilities' ? failureKind(step.error) : 'semantic' as FailureKind, message: `${step.step}: ${step.error ?? step.details ?? 'failed'}`, @@ -112,21 +134,92 @@ function assertDiscovery(result: DiscoveryResult): void { 'Discovery returned no supported version'); } +type ResolverOptions = Parameters[1]; + +/** Probe and storyboard-resolution options pinned to one release. */ +export interface ReleaseProbe { + options: TestOptions; + resolverOptions: ResolverOptions; +} + +/** Builds a probe pinned to the release the agent said it served. */ +export type RepinProbe = (release: string) => ReleaseProbe; + +const RELEASE_RE = /^([1-9]\d*)\.(\d+)(?:\.\d+)?(?:-([0-9A-Za-z.-]+))?$/; + +/** Release-precision wire value (`3.0`, `3.2-rc.7`) for a pin or an echo. */ +export function releasePrecision(version: unknown): string | undefined { + const match = typeof version === 'string' ? version.match(RELEASE_RE) : null; + return match ? `${match[1]}.${match[2]}${match[3] ? `-${match[3]}` : ''}` : undefined; +} + +/** The release the agent echoed on `get_adcp_capabilities`, if any. */ +function servedRelease(result: DiscoveryResult): string | undefined { + const raw = result.profile?.raw_capabilities as { adcp_version?: unknown } | undefined; + return releasePrecision(raw?.adcp_version); +} + +/** + * The SDK validates capabilities against the caller's pin, not the served + * release. A 3.0 pin cannot carry `adcp_version`, so the SDK sends only + * `adcp_major_version: 3` and the agent may serve its highest 3.x release + * (docs/reference/versioning.mdx, "Server resolves"). The spec says buyers + * validate against the echoed release, not their pin ("Server echoes"). + * + * When the echo differs from the pin, the agent must advertise the echoed + * release. The probe then discovers again pinned to it, so the SDK validates + * strictly against that release, and resolves storyboards from that + * release's bundle. The pinned probe must keep serving the same release. + */ +async function negotiateServedRelease( + url: string, + probe: ReleaseProbe, + result: DiscoveryResult, + discover: Discovery, + repin: RepinProbe | undefined, +): Promise<{ probe: ReleaseProbe; result: DiscoveryResult }> { + const served = servedRelease(result); + const pinned = releasePrecision(probe.options.wireAdcpVersion ?? probe.options.adcpVersion); + if (!served || !pinned || served === pinned || !repin) return { probe, result }; + const issues = result.profile?.capabilities_schema_issues; + const context = issues?.length ? `; issues against the ${pinned} pin: ${JSON.stringify(issues)}` : ''; + if (served.split('.')[0] !== pinned.split('.')[0]) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin; majors must agree${context}`); + } + if (!result.profile?.adcp_supported_versions?.includes(served)) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served}, which its supported_versions do not list${context}`); + } + let servedProbe: ReleaseProbe; + try { + servedProbe = repin(served); + } catch (error) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served}, but no hosted bundle is available to validate it: ${error instanceof Error ? error.message : String(error)}${context}`); + } + const servedResult = await discover(url, servedProbe.options); + const reserved = servedRelease(servedResult); + if (reserved !== undefined && reserved !== served) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin, but ${reserved} when pinned to ${served}`); + } + return { probe: servedProbe, result: servedResult }; +} + export async function checkProbe( url: string, options: TestOptions, - resolverOptions: Parameters[1], + resolverOptions: ResolverOptions, anonymous: boolean, { discover = (endpoint, opts) => boundedDiscovery(endpoint, opts), resolve = resolveStoryboardsForCapabilities, - }: { discover?: Discovery; resolve?: Resolver } = {}, + repin, + }: { discover?: Discovery; resolve?: Resolver; repin?: RepinProbe } = {}, ): Promise<'public' | 'authenticated' | 'auth-required'> { for (let attempt = 1; ; attempt++) { let result: DiscoveryResult; + let probe: ReleaseProbe = { options, resolverOptions }; try { - // Exactly one authenticated discovery per attempt; reuse its profile. - result = await discover(url, options); + // One discovery per negotiated release per attempt; reuse its profile. + ({ probe, result } = await negotiateServedRelease(url, probe, await discover(url, options), discover, repin)); assertDiscovery(result); } catch (error) { const kind = failureKind(error); @@ -140,7 +233,7 @@ export async function checkProbe( specialisms: profile.specialisms ?? [], major_versions: profile.adcp_major_versions, supported_versions: profile.adcp_supported_versions, - }, resolverOptions); + }, probe.resolverOptions); assert(resolved.storyboards.length > 0, 'No applicable storyboards'); assert(resolved.bundles.length > 0, 'No applicable bundles'); return anonymous ? 'public' : 'authenticated'; diff --git a/scripts/external-smoke/run.ts b/scripts/external-smoke/run.ts index df480be8b82..4025a402174 100644 --- a/scripts/external-smoke/run.ts +++ b/scripts/external-smoke/run.ts @@ -1,17 +1,29 @@ +import type { TestOptions } from '@adcp/sdk/testing'; import { PUBLIC_TEST_AGENT } from '../../server/src/config/test-agent.js'; import { hostedComplianceTarget, hostedComplianceOptions, withHostedTestOptions, } from '../../server/src/services/hosted-compliance-version.js'; -import { checkProbe } from './check.js'; +import { checkProbe, type RepinProbe } from './check.js'; const url = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url; const token = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token; const target = hostedComplianceTarget(); const resolverOptions = hostedComplianceOptions(target); +// When the agent serves a different release than the default pin, validate +// and resolve against this checkout's hosted bundle for the served release. +const repinFor = (base: Partial): RepinProbe => release => { + const served = hostedComplianceTarget(release); + return { options: withHostedTestOptions(base, served), resolverOptions: hostedComplianceOptions(served) }; +}; -await checkProbe(url, withHostedTestOptions({ auth: { type: 'bearer', token } }, target), resolverOptions, false); +const authenticated: Partial = { auth: { type: 'bearer', token } }; +await checkProbe(url, withHostedTestOptions(authenticated, target), resolverOptions, false, { + repin: repinFor(authenticated), +}); console.log('Authenticated discovery and storyboard resolution passed.'); -const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true); +const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true, { + repin: repinFor({}), +}); console.log(`Anonymous discovery: ${anonymous}.`); diff --git a/server/tests/unit/external-smoke.test.ts b/server/tests/unit/external-smoke.test.ts index 8f29afae437..006d64423da 100644 --- a/server/tests/unit/external-smoke.test.ts +++ b/server/tests/unit/external-smoke.test.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { parseDocument } from 'yaml'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { testCapabilityDiscovery, type TestOptions } from '@adcp/sdk/testing'; -import { boundedDiscovery, checkProbe, MAX_ATTEMPTS, SmokeFailure } from '../../../scripts/external-smoke/check.js'; +import { boundedDiscovery, checkProbe, MAX_ATTEMPTS, releasePrecision, SmokeFailure, type ReleaseProbe } from '../../../scripts/external-smoke/check.js'; const root = fileURLToPath(new URL('../../../', import.meta.url)); const url = 'https://smoke.example.com/mcp'; @@ -182,6 +182,138 @@ describe('external smoke discovery', () => { }); }); +describe('external smoke served-release negotiation', () => { + // A 3.0 pin cannot carry adcp_version, so the SDK sends only + // adcp_major_version: 3 and validates against 3.0. The agent may serve its + // highest 3.x release and echo it; the spec says to validate against that. + const pinned: TestOptions = { ...options, adcpVersion: '3.0.27' }; + const pinnedResolver = { version: '3.0.27' } as Parameters[2]; + const servedProbe: ReleaseProbe = { + options: { ...options, adcpVersion: '3.2.4' }, + resolverOptions: { version: '3.2.4' } as ReleaseProbe['resolverOptions'], + }; + const served = (echo: string | undefined, issues: Array<{ pointer: string; message: string }> = []): Result => { + const result = success(); + result.profile!.adcp_supported_versions = ['3.0', '3.1', '3.2']; + result.profile!.raw_capabilities = echo === undefined ? {} : { adcp_version: echo }; + if (issues.length) result.profile!.capabilities_schema_issues = issues; + return result; + }; + const threeTwoShapes = [ + { pointer: '/media_buy/features/bidding_policy', message: 'must be boolean' }, + { pointer: '/compliance_testing/scenarios/4', message: 'must be one of: "force_creative_status"' }, + ]; + + it('validates and resolves against the echoed release instead of the 3.0 pin', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2', threeTwoShapes)) + .mockResolvedValueOnce(served('3.2')); + const repin = vi.fn(() => servedProbe); + const resolve = vi.fn(() => resolved); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, resolve, repin })) + .resolves.toBe('authenticated'); + expect(repin).toHaveBeenCalledExactlyOnceWith('3.2'); + expect(discover).toHaveBeenNthCalledWith(1, url, pinned); + expect(discover).toHaveBeenNthCalledWith(2, url, servedProbe.options); + expect(resolve).toHaveBeenCalledExactlyOnceWith(expect.anything(), servedProbe.resolverOptions); + }); + + it('re-validates at the served release even when the pin validation passed', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2')) + .mockResolvedValueOnce(served('3.2', [{ pointer: '/adcp', message: 'invalid' }])); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin: () => servedProbe })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('/adcp') }); + expect(discover).toHaveBeenCalledTimes(2); + }); + + it.each([ + ['echoes the pinned release', '3.0'], + ['omits the echo', undefined], + ])('keeps pin validation when the agent %s', async (_label, echo) => { + const discover = vi.fn().mockResolvedValue(served(echo, threeTwoShapes)); + const repin = vi.fn(() => servedProbe); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('bidding_policy') }); + expect(discover).toHaveBeenCalledTimes(1); + expect(repin).not.toHaveBeenCalled(); + }); + + it.each([ + ['a release its supported_versions omit', '3.3', /supported_versions do not list/], + ['a different major', '4.0', /majors must agree/], + ])('rejects an agent that serves %s', async (_label, echo, message) => { + const discover = vi.fn().mockResolvedValue(served(echo, threeTwoShapes)); + const repin = vi.fn(() => servedProbe); + const rejection = checkProbe(url, pinned, pinnedResolver, false, { discover, repin }); + await expect(rejection).rejects.toMatchObject({ kind: 'semantic', message: expect.stringMatching(message) }); + await expect(rejection).rejects.toThrow(/bidding_policy/); + expect(discover).toHaveBeenCalledTimes(1); + expect(repin).not.toHaveBeenCalled(); + }); + + it('fails when no hosted bundle exists for the served release', async () => { + const discover = vi.fn().mockResolvedValue(served('3.2', threeTwoShapes)); + const repin = () => { throw new Error('No npm-published compliance version is registered for 3.2'); }; + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringMatching(/no hosted bundle.*3\.2.*bidding_policy/) }); + expect(discover).toHaveBeenCalledTimes(1); + }); + + it('fails when the agent serves another release once pinned to the echoed one', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2', threeTwoShapes)) + .mockResolvedValueOnce(served('3.1')); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin: () => servedProbe })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('but 3.1 when pinned to 3.2') }); + }); + + it('normalizes pins and echoes to release precision', () => { + expect(releasePrecision('3.0.27')).toBe('3.0'); + expect(releasePrecision('3.2')).toBe('3.2'); + expect(releasePrecision('3.2.0-rc.7')).toBe('3.2-rc.7'); + expect(releasePrecision('3.2-rc.7')).toBe('3.2-rc.7'); + expect(releasePrecision('v3')).toBeUndefined(); + expect(releasePrecision(3)).toBeUndefined(); + }); +}); + +describe('external smoke SDK cross-check and auth evidence', () => { + const complianceFalsePositive = 'Tools suggest unreported protocols: compliance (has: [comply_test_controller])'; + const withCrossCheck = (error: string, protocols: string[], declaresBlock = true): Result => { + const result = success(); + result.profile!.supported_protocols = protocols; + result.profile!.raw_capabilities = declaresBlock ? { compliance_testing: { scenarios: ['force_creative_status'] } } : {}; + result.steps.push({ step: 'Cross-validate protocols and tools', passed: false, duration_ms: 0, error }); + return result; + }; + + it('waives the SDK compliance cross-check false positive when compliance_testing was declared', async () => { + const discover = vi.fn().mockResolvedValue(withCrossCheck(complianceFalsePositive, ['media_buy', 'compliance_testing'])); + await expect(checkProbe(url, options, {}, false, { discover, resolve: () => resolved })).resolves.toBe('authenticated'); + }); + + it.each([ + ['compliance_testing was not declared', complianceFalsePositive, ['media_buy'], false], + ['the SDK synthesized compliance_testing without a declared block', complianceFalsePositive, ['media_buy', 'compliance_testing'], false], + ['the step reports another issue', `Protocol 'signals' reported but no matching tools found; ${complianceFalsePositive}`, ['media_buy', 'compliance_testing']], + ['another protocol is unreported', 'Tools suggest unreported protocols: signals (has: [get_signals])', ['media_buy', 'compliance_testing']], + ])('still fails the cross-check when %s', async (_label, error, protocols, declaresBlock = true) => { + const discover = vi.fn().mockResolvedValue(withCrossCheck(error, protocols, declaresBlock)); + await expect(checkProbe(url, options, {}, false, { discover, resolve: () => resolved })) + .rejects.toMatchObject({ kind: 'semantic' }); + }); + + it('treats an SDK OAuth challenge as anonymous auth-required evidence', async () => { + const discover = vi.fn().mockResolvedValue(failure( + 'Agent https://smoke.example.com/mcp requires OAuth authorization. Authorization server: https://auth.example.com. Provide an OAuthFlowHandler or run an interactive flow to complete authorization.', + )); + await expect(checkProbe(url, {}, {}, true, { discover })).resolves.toBe('auth-required'); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toMatchObject({ kind: 'auth' }); + expect(discover).toHaveBeenCalledTimes(2); + }); +}); + describe('external smoke CI isolation', () => { it('default integration selection cannot execute the external harness', () => { const pkg = JSON.parse(readFileSync(resolvePath(root, 'package.json'), 'utf8'));