From 45013348548b10671ff23902812186c754fb2b45 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 15 Oct 2026 09:43:17 +0000 Subject: [PATCH] fix(ci): validate external smoke capabilities against the served release The daily external smoke job has failed every run since #7511. It pins the default hosted line (3.0). A 3.0 pin cannot carry adcp_version, so @adcp/sdk sends get_adcp_capabilities with only adcp_major_version: 3. Per docs/reference/versioning.mdx the agent serves its highest 3.x release and echoes adcp_version "3.2", but testCapabilityDiscovery validates the response against the 3.0 pin, so 3.2-only shapes (features.bidding_policy, newer compliance_testing scenarios) are reported as schema errors. The spec says buyers validate against the echoed release, not their pin. When the echo differs from the pin, the check now requires the echo to be in adcp.supported_versions, discovers again pinned to that release (the SDK then validates strictly against it), requires the agent to keep serving it, and resolves storyboards from that release's hosted bundle. Two more failures were hidden behind the schema error: - SDK 15.8.1 keys comply_test_controller as protocol "compliance" in its tool cross-check but parses the compliance_testing block as "compliance_testing", so every compliance-capable agent fails that step. Waive only that exact message, and only when the response carries the block. - The anonymous probe now gets a 401 Bearer challenge. Classify the SDK's "requires OAuth authorization" message as auth evidence. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/external-smoke/README.md | 22 +++- scripts/external-smoke/check.ts | 107 ++++++++++++++++-- scripts/external-smoke/run.ts | 18 ++- server/tests/unit/external-smoke.test.ts | 134 ++++++++++++++++++++++- 4 files changed, 267 insertions(+), 14 deletions(-) diff --git a/scripts/external-smoke/README.md b/scripts/external-smoke/README.md index ad5d4cad8cd..1f2a1304640 100644 --- a/scripts/external-smoke/README.md +++ b/scripts/external-smoke/README.md @@ -10,9 +10,25 @@ Each authenticated/anonymous probe makes at most two discovery attempts, each with a fresh AbortSignal and fetch closure, a 10s total budget, and explicit 4s SDK transport timeout. Only categorized transient transport failures retry; schema, assertion, auth and unknown errors do not. -Authenticated discovery runs once per attempt and its profile is reused for -local storyboard resolution. Anonymous success meets the same checks, or -failed discovery must supply concrete authentication error evidence. +Authenticated discovery runs once per negotiated release per attempt, and its +profile is reused for local storyboard resolution. Anonymous success meets the +same checks, or failed discovery must supply concrete authentication error +evidence (including the SDK's OAuth challenge, "requires OAuth authorization"). + +The probe pins the default hosted line (3.0). A 3.0 pin cannot carry +`adcp_version`, so the SDK sends only `adcp_major_version: 3`, and the agent +may serve and echo its highest 3.x release. `@adcp/sdk` validates the response +against the 3.0 pin regardless. The versioning spec says buyers validate +against the echoed release. When the echo differs from the pin, the smoke check +therefore requires the echo to be listed in `adcp.supported_versions`, +discovers again pinned to that release, validates strictly against it, and +resolves storyboards from that release's hosted bundle. An echo with no hosted +bundle, a different major, or a different release on the pinned probe fails. + +SDK 15.8.1 also keys `comply_test_controller` as protocol `compliance` in its +tool cross-check, while it parses the `compliance_testing` block as +`compliance_testing`. The check waives only that exact cross-check message, +and only when the response carries the `compliance_testing` block. SDK 14.0.0-rc.35 does not pass the signal to its second capability call. The local `trustedFetchFn` composes the attempt signal with the transport signal diff --git a/scripts/external-smoke/check.ts b/scripts/external-smoke/check.ts index 70d67380233..a807df44f6d 100644 --- a/scripts/external-smoke/check.ts +++ b/scripts/external-smoke/check.ts @@ -23,13 +23,16 @@ export class SmokeFailure extends Error { // SDK step errors are strings. Keep this allowlist narrow; unknown errors fail // once. In particular, a generic mention of "auth" or "network" proves nothing. +// `requires OAuth authorization` is the message of the SDK's +// NeedsAuthorizationError (a 401 Bearer challenge), which reaches us only as a +// step string. function failureKind(error: unknown): FailureKind { if (error instanceof SmokeFailure) return error.kind; const value = error as { name?: string; code?: string; status?: number; message?: string; cause?: unknown } | null; const evidence = typeof error === 'string' ? error : [value?.name, value?.code, value?.status, value?.message].join(' '); if (/\b(?:AssertionError|ZodError|SchemaValidationError)\b|schema|validation|unsupported version/i.test(evidence)) return 'semantic'; - if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token/i.test(evidence)) return 'auth'; + if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token|\brequires OAuth authorization\b/i.test(evidence)) return 'auth'; if ([408, 429, 502, 503, 504].includes(value?.status ?? 0) || /\b(?:ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET|TimeoutError|SmokeTransientTransportError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:408|429|502|503|504)\b|request timed out|request timeout|version negotiation probe timed out/i.test(evidence)) return 'transient'; return value?.cause ? failureKind(value.cause) : 'semantic'; } @@ -85,10 +88,29 @@ export async function boundedDiscovery( } } +// @adcp/sdk 15.8.1 (and 15.9.1) parses a `compliance_testing` capability +// block as protocol `compliance_testing`, but its tool cross-check keys +// comply_test_controller under `compliance`. Every agent that correctly +// declares the block therefore fails the step with exactly this message. +// Waive only that exact message, and only when the agent's response carries +// the block (the SDK also adds the protocol when the block is missing). Any +// other cross-check issue still fails. +const SDK_COMPLIANCE_CROSS_CHECK_STEP = 'Cross-validate protocols and tools'; +const SDK_COMPLIANCE_CROSS_CHECK_ERROR = 'Tools suggest unreported protocols: compliance (has: [comply_test_controller])'; + +function isSdkComplianceCrossCheckFalsePositive(step: DiscoveryResult['steps'][number], result: DiscoveryResult): boolean { + return step.step === SDK_COMPLIANCE_CROSS_CHECK_STEP + && step.error === SDK_COMPLIANCE_CROSS_CHECK_ERROR + && result.profile?.supported_protocols?.includes('compliance_testing') === true + && typeof (result.profile.raw_capabilities as { compliance_testing?: unknown } | undefined)?.compliance_testing === 'object'; +} + function assertDiscovery(result: DiscoveryResult): void { const schemaIssues = result.profile?.capabilities_schema_issues; if (schemaIssues?.length) throw new SmokeFailure('semantic', `Capability schema errors: ${JSON.stringify(schemaIssues)}`); - const failures = result.steps.filter(step => !step.passed && !step.skipped).map(step => ({ + const failed = result.steps.filter(step => !step.passed && !step.skipped + && !isSdkComplianceCrossCheckFalsePositive(step, result)); + const failures = failed.map(step => ({ kind: step.task === 'getAgentInfo' || step.task === 'get_adcp_capabilities' ? failureKind(step.error) : 'semantic' as FailureKind, message: `${step.step}: ${step.error ?? step.details ?? 'failed'}`, @@ -112,21 +134,92 @@ function assertDiscovery(result: DiscoveryResult): void { 'Discovery returned no supported version'); } +type ResolverOptions = Parameters[1]; + +/** Probe and storyboard-resolution options pinned to one release. */ +export interface ReleaseProbe { + options: TestOptions; + resolverOptions: ResolverOptions; +} + +/** Builds a probe pinned to the release the agent said it served. */ +export type RepinProbe = (release: string) => ReleaseProbe; + +const RELEASE_RE = /^([1-9]\d*)\.(\d+)(?:\.\d+)?(?:-([0-9A-Za-z.-]+))?$/; + +/** Release-precision wire value (`3.0`, `3.2-rc.7`) for a pin or an echo. */ +export function releasePrecision(version: unknown): string | undefined { + const match = typeof version === 'string' ? version.match(RELEASE_RE) : null; + return match ? `${match[1]}.${match[2]}${match[3] ? `-${match[3]}` : ''}` : undefined; +} + +/** The release the agent echoed on `get_adcp_capabilities`, if any. */ +function servedRelease(result: DiscoveryResult): string | undefined { + const raw = result.profile?.raw_capabilities as { adcp_version?: unknown } | undefined; + return releasePrecision(raw?.adcp_version); +} + +/** + * The SDK validates capabilities against the caller's pin, not the served + * release. A 3.0 pin cannot carry `adcp_version`, so the SDK sends only + * `adcp_major_version: 3` and the agent may serve its highest 3.x release + * (docs/reference/versioning.mdx, "Server resolves"). The spec says buyers + * validate against the echoed release, not their pin ("Server echoes"). + * + * When the echo differs from the pin, the agent must advertise the echoed + * release. The probe then discovers again pinned to it, so the SDK validates + * strictly against that release, and resolves storyboards from that + * release's bundle. The pinned probe must keep serving the same release. + */ +async function negotiateServedRelease( + url: string, + probe: ReleaseProbe, + result: DiscoveryResult, + discover: Discovery, + repin: RepinProbe | undefined, +): Promise<{ probe: ReleaseProbe; result: DiscoveryResult }> { + const served = servedRelease(result); + const pinned = releasePrecision(probe.options.wireAdcpVersion ?? probe.options.adcpVersion); + if (!served || !pinned || served === pinned || !repin) return { probe, result }; + const issues = result.profile?.capabilities_schema_issues; + const context = issues?.length ? `; issues against the ${pinned} pin: ${JSON.stringify(issues)}` : ''; + if (served.split('.')[0] !== pinned.split('.')[0]) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin; majors must agree${context}`); + } + if (!result.profile?.adcp_supported_versions?.includes(served)) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served}, which its supported_versions do not list${context}`); + } + let servedProbe: ReleaseProbe; + try { + servedProbe = repin(served); + } catch (error) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served}, but no hosted bundle is available to validate it: ${error instanceof Error ? error.message : String(error)}${context}`); + } + const servedResult = await discover(url, servedProbe.options); + const reserved = servedRelease(servedResult); + if (reserved !== undefined && reserved !== served) { + throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin, but ${reserved} when pinned to ${served}`); + } + return { probe: servedProbe, result: servedResult }; +} + export async function checkProbe( url: string, options: TestOptions, - resolverOptions: Parameters[1], + resolverOptions: ResolverOptions, anonymous: boolean, { discover = (endpoint, opts) => boundedDiscovery(endpoint, opts), resolve = resolveStoryboardsForCapabilities, - }: { discover?: Discovery; resolve?: Resolver } = {}, + repin, + }: { discover?: Discovery; resolve?: Resolver; repin?: RepinProbe } = {}, ): Promise<'public' | 'authenticated' | 'auth-required'> { for (let attempt = 1; ; attempt++) { let result: DiscoveryResult; + let probe: ReleaseProbe = { options, resolverOptions }; try { - // Exactly one authenticated discovery per attempt; reuse its profile. - result = await discover(url, options); + // One discovery per negotiated release per attempt; reuse its profile. + ({ probe, result } = await negotiateServedRelease(url, probe, await discover(url, options), discover, repin)); assertDiscovery(result); } catch (error) { const kind = failureKind(error); @@ -140,7 +233,7 @@ export async function checkProbe( specialisms: profile.specialisms ?? [], major_versions: profile.adcp_major_versions, supported_versions: profile.adcp_supported_versions, - }, resolverOptions); + }, probe.resolverOptions); assert(resolved.storyboards.length > 0, 'No applicable storyboards'); assert(resolved.bundles.length > 0, 'No applicable bundles'); return anonymous ? 'public' : 'authenticated'; diff --git a/scripts/external-smoke/run.ts b/scripts/external-smoke/run.ts index df480be8b82..4025a402174 100644 --- a/scripts/external-smoke/run.ts +++ b/scripts/external-smoke/run.ts @@ -1,17 +1,29 @@ +import type { TestOptions } from '@adcp/sdk/testing'; import { PUBLIC_TEST_AGENT } from '../../server/src/config/test-agent.js'; import { hostedComplianceTarget, hostedComplianceOptions, withHostedTestOptions, } from '../../server/src/services/hosted-compliance-version.js'; -import { checkProbe } from './check.js'; +import { checkProbe, type RepinProbe } from './check.js'; const url = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url; const token = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token; const target = hostedComplianceTarget(); const resolverOptions = hostedComplianceOptions(target); +// When the agent serves a different release than the default pin, validate +// and resolve against this checkout's hosted bundle for the served release. +const repinFor = (base: Partial): RepinProbe => release => { + const served = hostedComplianceTarget(release); + return { options: withHostedTestOptions(base, served), resolverOptions: hostedComplianceOptions(served) }; +}; -await checkProbe(url, withHostedTestOptions({ auth: { type: 'bearer', token } }, target), resolverOptions, false); +const authenticated: Partial = { auth: { type: 'bearer', token } }; +await checkProbe(url, withHostedTestOptions(authenticated, target), resolverOptions, false, { + repin: repinFor(authenticated), +}); console.log('Authenticated discovery and storyboard resolution passed.'); -const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true); +const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true, { + repin: repinFor({}), +}); console.log(`Anonymous discovery: ${anonymous}.`); diff --git a/server/tests/unit/external-smoke.test.ts b/server/tests/unit/external-smoke.test.ts index 8f29afae437..006d64423da 100644 --- a/server/tests/unit/external-smoke.test.ts +++ b/server/tests/unit/external-smoke.test.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { parseDocument } from 'yaml'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { testCapabilityDiscovery, type TestOptions } from '@adcp/sdk/testing'; -import { boundedDiscovery, checkProbe, MAX_ATTEMPTS, SmokeFailure } from '../../../scripts/external-smoke/check.js'; +import { boundedDiscovery, checkProbe, MAX_ATTEMPTS, releasePrecision, SmokeFailure, type ReleaseProbe } from '../../../scripts/external-smoke/check.js'; const root = fileURLToPath(new URL('../../../', import.meta.url)); const url = 'https://smoke.example.com/mcp'; @@ -182,6 +182,138 @@ describe('external smoke discovery', () => { }); }); +describe('external smoke served-release negotiation', () => { + // A 3.0 pin cannot carry adcp_version, so the SDK sends only + // adcp_major_version: 3 and validates against 3.0. The agent may serve its + // highest 3.x release and echo it; the spec says to validate against that. + const pinned: TestOptions = { ...options, adcpVersion: '3.0.27' }; + const pinnedResolver = { version: '3.0.27' } as Parameters[2]; + const servedProbe: ReleaseProbe = { + options: { ...options, adcpVersion: '3.2.4' }, + resolverOptions: { version: '3.2.4' } as ReleaseProbe['resolverOptions'], + }; + const served = (echo: string | undefined, issues: Array<{ pointer: string; message: string }> = []): Result => { + const result = success(); + result.profile!.adcp_supported_versions = ['3.0', '3.1', '3.2']; + result.profile!.raw_capabilities = echo === undefined ? {} : { adcp_version: echo }; + if (issues.length) result.profile!.capabilities_schema_issues = issues; + return result; + }; + const threeTwoShapes = [ + { pointer: '/media_buy/features/bidding_policy', message: 'must be boolean' }, + { pointer: '/compliance_testing/scenarios/4', message: 'must be one of: "force_creative_status"' }, + ]; + + it('validates and resolves against the echoed release instead of the 3.0 pin', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2', threeTwoShapes)) + .mockResolvedValueOnce(served('3.2')); + const repin = vi.fn(() => servedProbe); + const resolve = vi.fn(() => resolved); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, resolve, repin })) + .resolves.toBe('authenticated'); + expect(repin).toHaveBeenCalledExactlyOnceWith('3.2'); + expect(discover).toHaveBeenNthCalledWith(1, url, pinned); + expect(discover).toHaveBeenNthCalledWith(2, url, servedProbe.options); + expect(resolve).toHaveBeenCalledExactlyOnceWith(expect.anything(), servedProbe.resolverOptions); + }); + + it('re-validates at the served release even when the pin validation passed', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2')) + .mockResolvedValueOnce(served('3.2', [{ pointer: '/adcp', message: 'invalid' }])); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin: () => servedProbe })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('/adcp') }); + expect(discover).toHaveBeenCalledTimes(2); + }); + + it.each([ + ['echoes the pinned release', '3.0'], + ['omits the echo', undefined], + ])('keeps pin validation when the agent %s', async (_label, echo) => { + const discover = vi.fn().mockResolvedValue(served(echo, threeTwoShapes)); + const repin = vi.fn(() => servedProbe); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('bidding_policy') }); + expect(discover).toHaveBeenCalledTimes(1); + expect(repin).not.toHaveBeenCalled(); + }); + + it.each([ + ['a release its supported_versions omit', '3.3', /supported_versions do not list/], + ['a different major', '4.0', /majors must agree/], + ])('rejects an agent that serves %s', async (_label, echo, message) => { + const discover = vi.fn().mockResolvedValue(served(echo, threeTwoShapes)); + const repin = vi.fn(() => servedProbe); + const rejection = checkProbe(url, pinned, pinnedResolver, false, { discover, repin }); + await expect(rejection).rejects.toMatchObject({ kind: 'semantic', message: expect.stringMatching(message) }); + await expect(rejection).rejects.toThrow(/bidding_policy/); + expect(discover).toHaveBeenCalledTimes(1); + expect(repin).not.toHaveBeenCalled(); + }); + + it('fails when no hosted bundle exists for the served release', async () => { + const discover = vi.fn().mockResolvedValue(served('3.2', threeTwoShapes)); + const repin = () => { throw new Error('No npm-published compliance version is registered for 3.2'); }; + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringMatching(/no hosted bundle.*3\.2.*bidding_policy/) }); + expect(discover).toHaveBeenCalledTimes(1); + }); + + it('fails when the agent serves another release once pinned to the echoed one', async () => { + const discover = vi.fn() + .mockResolvedValueOnce(served('3.2', threeTwoShapes)) + .mockResolvedValueOnce(served('3.1')); + await expect(checkProbe(url, pinned, pinnedResolver, false, { discover, repin: () => servedProbe })) + .rejects.toMatchObject({ kind: 'semantic', message: expect.stringContaining('but 3.1 when pinned to 3.2') }); + }); + + it('normalizes pins and echoes to release precision', () => { + expect(releasePrecision('3.0.27')).toBe('3.0'); + expect(releasePrecision('3.2')).toBe('3.2'); + expect(releasePrecision('3.2.0-rc.7')).toBe('3.2-rc.7'); + expect(releasePrecision('3.2-rc.7')).toBe('3.2-rc.7'); + expect(releasePrecision('v3')).toBeUndefined(); + expect(releasePrecision(3)).toBeUndefined(); + }); +}); + +describe('external smoke SDK cross-check and auth evidence', () => { + const complianceFalsePositive = 'Tools suggest unreported protocols: compliance (has: [comply_test_controller])'; + const withCrossCheck = (error: string, protocols: string[], declaresBlock = true): Result => { + const result = success(); + result.profile!.supported_protocols = protocols; + result.profile!.raw_capabilities = declaresBlock ? { compliance_testing: { scenarios: ['force_creative_status'] } } : {}; + result.steps.push({ step: 'Cross-validate protocols and tools', passed: false, duration_ms: 0, error }); + return result; + }; + + it('waives the SDK compliance cross-check false positive when compliance_testing was declared', async () => { + const discover = vi.fn().mockResolvedValue(withCrossCheck(complianceFalsePositive, ['media_buy', 'compliance_testing'])); + await expect(checkProbe(url, options, {}, false, { discover, resolve: () => resolved })).resolves.toBe('authenticated'); + }); + + it.each([ + ['compliance_testing was not declared', complianceFalsePositive, ['media_buy'], false], + ['the SDK synthesized compliance_testing without a declared block', complianceFalsePositive, ['media_buy', 'compliance_testing'], false], + ['the step reports another issue', `Protocol 'signals' reported but no matching tools found; ${complianceFalsePositive}`, ['media_buy', 'compliance_testing']], + ['another protocol is unreported', 'Tools suggest unreported protocols: signals (has: [get_signals])', ['media_buy', 'compliance_testing']], + ])('still fails the cross-check when %s', async (_label, error, protocols, declaresBlock = true) => { + const discover = vi.fn().mockResolvedValue(withCrossCheck(error, protocols, declaresBlock)); + await expect(checkProbe(url, options, {}, false, { discover, resolve: () => resolved })) + .rejects.toMatchObject({ kind: 'semantic' }); + }); + + it('treats an SDK OAuth challenge as anonymous auth-required evidence', async () => { + const discover = vi.fn().mockResolvedValue(failure( + 'Agent https://smoke.example.com/mcp requires OAuth authorization. Authorization server: https://auth.example.com. Provide an OAuthFlowHandler or run an interactive flow to complete authorization.', + )); + await expect(checkProbe(url, {}, {}, true, { discover })).resolves.toBe('auth-required'); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toMatchObject({ kind: 'auth' }); + expect(discover).toHaveBeenCalledTimes(2); + }); +}); + describe('external smoke CI isolation', () => { it('default integration selection cannot execute the external harness', () => { const pkg = JSON.parse(readFileSync(resolvePath(root, 'package.json'), 'utf8'));