diff --git a/.github/workflows/external-smoke.yml b/.github/workflows/external-smoke.yml new file mode 100644 index 00000000000..d6e39977200 --- /dev/null +++ b/.github/workflows/external-smoke.yml @@ -0,0 +1,32 @@ +name: External smoke + +# Synthetic monitoring of the deployed public agent; not candidate validation +# or a release gate. Keep live requests out of required PR/push test shards. +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: external-smoke + cancel-in-progress: false + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '24' + cache: npm + - run: npm ci + env: + PUPPETEER_SKIP_DOWNLOAD: 'true' + - name: Check deployed public agent + timeout-minutes: 2 + run: npm run test:external-smoke diff --git a/package.json b/package.json index be6bd6c2f15..40c3b98aaba 100644 --- a/package.json +++ b/package.json @@ -138,6 +138,7 @@ "test:redteam": "tsx server/src/addie/testing/redteam-cli.ts", "test:server-unit": "vitest run --config server/vitest.config.ts tests/unit/", "test:server-integration": "vitest run server/tests/integration --config server/vitest.config.ts", + "test:external-smoke": "tsx scripts/external-smoke/run.ts", "test:openapi": "tsx scripts/generate-openapi.ts && git diff --exit-code static/openapi/registry.yaml || (echo 'OpenAPI spec is out of date. Run: npm run build:openapi' && exit 1)", "test:registry": "vitest run server/tests --exclude 'server/tests/simulation/**'", "test:simulation": "vitest run server/tests/simulation/scenarios --pool forks --poolOptions.forks.singleFork --testTimeout 120000", diff --git a/scripts/external-smoke/README.md b/scripts/external-smoke/README.md new file mode 100644 index 00000000000..ad5d4cad8cd --- /dev/null +++ b/scripts/external-smoke/README.md @@ -0,0 +1,25 @@ +# External smoke + +`npm run test:external-smoke` checks the deployed public test agent using the +hosted compliance artifacts in this checkout. Override `ADCP_SMOKE_CHECK_URL` +and `ADCP_SMOKE_CHECK_TOKEN` for an operator-selected endpoint. This is synthetic +monitoring, not validation of candidate server code or a release gate. The +scheduled/manual `external-smoke.yml` workflow is its only CI caller. + +Each authenticated/anonymous probe makes at most two discovery attempts, each +with a fresh AbortSignal and fetch closure, a 10s total +budget, and explicit 4s SDK transport timeout. Only categorized transient +transport failures retry; schema, assertion, auth and unknown errors do not. +Authenticated discovery runs once per attempt and its profile is reused for +local storyboard resolution. Anonymous success meets the same checks, or +failed discovery must supply concrete authentication error evidence. + +SDK 14.0.0-rc.35 does not pass the signal to its second capability call. The +local `trustedFetchFn` composes the attempt signal with the transport signal +and rejects new requests after abort, including late SDK calls. Native fetch +aborts in-flight requests and response bodies. A deadline race bounds the +caller, but cannot cancel SDK JavaScript or a custom fetch implementation that +ignores abort; those promises may continue unwinding. SDK connection cache +entries are isolated by fetch identity and may remain until the runner exits. +This test-only trusted fetch uses Node's networking for +operator-selected endpoints; it is not a hosted untrusted-URL security boundary. diff --git a/scripts/external-smoke/check.ts b/scripts/external-smoke/check.ts new file mode 100644 index 00000000000..70d67380233 --- /dev/null +++ b/scripts/external-smoke/check.ts @@ -0,0 +1,148 @@ +import assert from 'node:assert/strict'; +import { + testCapabilityDiscovery, + resolveStoryboardsForCapabilities, + type TestOptions, +} from '@adcp/sdk/testing'; + +export const MAX_ATTEMPTS = 2; +export const ATTEMPT_TIMEOUT_MS = 10_000; +export const REQUEST_TIMEOUT_MS = 4_000; + +type Discovery = typeof testCapabilityDiscovery; +type DiscoveryResult = Awaited>; +type Resolver = typeof resolveStoryboardsForCapabilities; +type FailureKind = 'auth' | 'transient' | 'semantic'; + +export class SmokeFailure extends Error { + constructor(readonly kind: FailureKind, message: string) { + super(message); + this.name = 'SmokeFailure'; + } +} + +// SDK step errors are strings. Keep this allowlist narrow; unknown errors fail +// once. In particular, a generic mention of "auth" or "network" proves nothing. +function failureKind(error: unknown): FailureKind { + if (error instanceof SmokeFailure) return error.kind; + const value = error as { name?: string; code?: string; status?: number; message?: string; cause?: unknown } | null; + const evidence = typeof error === 'string' ? error : + [value?.name, value?.code, value?.status, value?.message].join(' '); + if (/\b(?:AssertionError|ZodError|SchemaValidationError)\b|schema|validation|unsupported version/i.test(evidence)) return 'semantic'; + if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token/i.test(evidence)) return 'auth'; + if ([408, 429, 502, 503, 504].includes(value?.status ?? 0) || /\b(?:ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET|TimeoutError|SmokeTransientTransportError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:408|429|502|503|504)\b|request timed out|request timeout|version negotiation probe timed out/i.test(evidence)) return 'transient'; + return value?.cause ? failureKind(value.cause) : 'semantic'; +} + +/** Abort the actual fetch boundary, including SDK calls that omit options.signal. */ +export async function boundedDiscovery( + url: string, + options: TestOptions, + { + discover = testCapabilityDiscovery, + fetchFn = fetch, + attemptTimeoutMs = ATTEMPT_TIMEOUT_MS, + requestTimeoutMs = REQUEST_TIMEOUT_MS, + }: { discover?: Discovery; fetchFn?: typeof fetch; attemptTimeoutMs?: number; requestTimeoutMs?: number } = {}, +): Promise { + const controller = new AbortController(); + const timeout = new SmokeFailure('transient', `Discovery attempt timed out after ${attemptTimeoutMs}ms`); + const timer = setTimeout(() => controller.abort(timeout), attemptTimeoutMs); + const scopedFetch: typeof fetch = async (input, init) => { + controller.signal.throwIfAborted(); + const signals = [controller.signal, init?.signal, input instanceof Request ? input.signal : undefined] + .filter((signal): signal is AbortSignal => signal != null); + try { + return await fetchFn(input, { ...init, signal: AbortSignal.any(signals) }); + } catch (error) { + // Preserve categorized cause evidence before rc.35 reduces it to a string. + const kind = failureKind(error); + if (kind === 'transient') throw new SmokeFailure(kind, `SmokeTransientTransportError: ${String(error)}`); + throw error; + } + }; + let onAbort: () => void = () => {}; + const aborted = new Promise((_, reject) => { + onAbort = () => reject(controller.signal.reason); + controller.signal.addEventListener('abort', onAbort, { once: true }); + }); + try { + // rc.35 keys connections by fetch identity, so attempts cannot reuse them. + return await Promise.race([ + discover(url, { + ...options, + signal: controller.signal, + transport: { ...options.transport, requestTimeoutMs, trustedFetchFn: scopedFetch }, + }), + aborted, + ]); + } finally { + clearTimeout(timer); + controller.signal.removeEventListener('abort', onAbort); + // Also close pending response bodies / late SDK calls after success or error. + // SDK promises can still unwind; this does not claim to cancel JavaScript. + controller.abort(new Error('Smoke attempt finished')); + } +} + +function assertDiscovery(result: DiscoveryResult): void { + const schemaIssues = result.profile?.capabilities_schema_issues; + if (schemaIssues?.length) throw new SmokeFailure('semantic', `Capability schema errors: ${JSON.stringify(schemaIssues)}`); + const failures = result.steps.filter(step => !step.passed && !step.skipped).map(step => ({ + kind: step.task === 'getAgentInfo' || step.task === 'get_adcp_capabilities' + ? failureKind(step.error) : 'semantic' as FailureKind, + message: `${step.step}: ${step.error ?? step.details ?? 'failed'}`, + })); + if (result.profile?.capabilities_probe_error) failures.push({ + kind: failureKind(result.profile.capabilities_probe_error), + message: result.profile.capabilities_probe_error, + }); + if (failures.length) { + // Mixed semantic/auth + transient failures must never turn into a retry. + const kind = failures.some(f => f.kind === 'semantic') ? 'semantic' : + failures.some(f => f.kind === 'auth') ? 'auth' : 'transient'; + throw new SmokeFailure(kind, failures.map(f => f.message).join('; ')); + } + assert(result.steps.some(step => step.task === 'getAgentInfo' && step.passed && !step.skipped), 'No passed discovery step'); + assert(result.profile?.tools.length, 'Discovery returned no tools'); + const versions = result.profile.adcp_supported_versions; + assert(versions === undefined + ? ['v2', 'v3'].includes(result.profile.adcp_version ?? '') + : versions.length > 0 && versions.every(version => /^\d+\.\d+(?:\.\d+)?(?:-[\w.-]+)?$/.test(version)), + 'Discovery returned no supported version'); +} + +export async function checkProbe( + url: string, + options: TestOptions, + resolverOptions: Parameters[1], + anonymous: boolean, + { + discover = (endpoint, opts) => boundedDiscovery(endpoint, opts), + resolve = resolveStoryboardsForCapabilities, + }: { discover?: Discovery; resolve?: Resolver } = {}, +): Promise<'public' | 'authenticated' | 'auth-required'> { + for (let attempt = 1; ; attempt++) { + let result: DiscoveryResult; + try { + // Exactly one authenticated discovery per attempt; reuse its profile. + result = await discover(url, options); + assertDiscovery(result); + } catch (error) { + const kind = failureKind(error); + if (anonymous && kind === 'auth') return 'auth-required'; + if (kind !== 'transient' || attempt === MAX_ATTEMPTS) throw error; + continue; + } + const profile = result.profile!; + const resolved = resolve({ + supported_protocols: profile.supported_protocols ?? [], + specialisms: profile.specialisms ?? [], + major_versions: profile.adcp_major_versions, + supported_versions: profile.adcp_supported_versions, + }, resolverOptions); + assert(resolved.storyboards.length > 0, 'No applicable storyboards'); + assert(resolved.bundles.length > 0, 'No applicable bundles'); + return anonymous ? 'public' : 'authenticated'; + } +} diff --git a/scripts/external-smoke/run.ts b/scripts/external-smoke/run.ts new file mode 100644 index 00000000000..df480be8b82 --- /dev/null +++ b/scripts/external-smoke/run.ts @@ -0,0 +1,17 @@ +import { PUBLIC_TEST_AGENT } from '../../server/src/config/test-agent.js'; +import { + hostedComplianceTarget, + hostedComplianceOptions, + withHostedTestOptions, +} from '../../server/src/services/hosted-compliance-version.js'; +import { checkProbe } from './check.js'; + +const url = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url; +const token = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token; +const target = hostedComplianceTarget(); +const resolverOptions = hostedComplianceOptions(target); + +await checkProbe(url, withHostedTestOptions({ auth: { type: 'bearer', token } }, target), resolverOptions, false); +console.log('Authenticated discovery and storyboard resolution passed.'); +const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true); +console.log(`Anonymous discovery: ${anonymous}.`); diff --git a/server/tests/integration/external-smoke-check.test.ts b/server/tests/integration/external-smoke-check.test.ts deleted file mode 100644 index fe672533bc2..00000000000 --- a/server/tests/integration/external-smoke-check.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * External-endpoint smoke check for release gate. - * - * Verifies the hosted verifier can discover, resolve, and classify errors - * for a real external agent. Uses the public test agent by default; override - * via ADCP_SMOKE_CHECK_URL / ADCP_SMOKE_CHECK_TOKEN. - * - * Run: - * npx vitest run server/tests/integration/external-smoke-check.test.ts - * - * Against a custom endpoint: - * ADCP_SMOKE_CHECK_URL=https://example.com/mcp \ - * ADCP_SMOKE_CHECK_TOKEN=sk-... \ - * npx vitest run server/tests/integration/external-smoke-check.test.ts - */ - -import { describe, it, expect } from 'vitest'; -import { - testCapabilityDiscovery, - resolveStoryboardsForCapabilities, -} from '@adcp/sdk/testing'; -import { PUBLIC_TEST_AGENT } from '../../src/config/test-agent.js'; -import { - hostedComplianceTarget, - hostedComplianceOptions, - withHostedTestOptions, -} from '../../src/services/hosted-compliance-version.js'; - -const SMOKE_URL = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url; -const SMOKE_TOKEN = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token; -const complianceTarget = hostedComplianceTarget(); -const complianceOptions = hostedComplianceOptions(complianceTarget); - -describe('external endpoint smoke check', () => { - it('discovers capabilities from external endpoint', async () => { - const result = await testCapabilityDiscovery(SMOKE_URL, withHostedTestOptions({ - auth: { type: 'bearer', token: SMOKE_TOKEN }, - }, complianceTarget)); - - expect(result.profile).toBeDefined(); - expect(result.profile!.tools!.length).toBeGreaterThan(0); - expect(result.profile!.adcp_supported_versions ?? result.profile!.adcp_version).toBeDefined(); - }, 30_000); - - it('resolves applicable storyboards for discovered capabilities', async () => { - const caps = await testCapabilityDiscovery(SMOKE_URL, withHostedTestOptions({ - auth: { type: 'bearer', token: SMOKE_TOKEN }, - }, complianceTarget)); - - const profile = caps.profile!; - const resolved = resolveStoryboardsForCapabilities({ - supported_protocols: profile.supported_protocols ?? [], - specialisms: profile.specialisms ?? [], - major_versions: profile.adcp_major_versions, - supported_versions: profile.adcp_supported_versions, - }, complianceOptions); - - expect(resolved.storyboards.length).toBeGreaterThan(0); - expect(resolved.bundles.length).toBeGreaterThan(0); - }, 30_000); - - it('classifies auth-required agents distinctly from discovery failure', async () => { - const unauthResult = await testCapabilityDiscovery(SMOKE_URL, withHostedTestOptions({ - // no auth — anonymous probe - }, complianceTarget)); - - if (unauthResult.profile) { - // Public agent — anon discovery succeeds, no auth error to classify - expect(unauthResult.profile.tools).toBeDefined(); - } else { - // Auth-gated agent — steps should show auth failure, not generic crash - const failedSteps = unauthResult.steps.filter(s => !s.passed); - expect(failedSteps.length).toBeGreaterThan(0); - const hasAuthIndicator = failedSteps.some(s => - /auth|401|unauthorized|forbidden/i.test(s.error ?? '') || - /auth|401|unauthorized|forbidden/i.test(s.step_id ?? ''), - ); - expect(hasAuthIndicator).toBe(true); - } - }, 30_000); -}); diff --git a/server/tests/unit/external-smoke.test.ts b/server/tests/unit/external-smoke.test.ts new file mode 100644 index 00000000000..8f29afae437 --- /dev/null +++ b/server/tests/unit/external-smoke.test.ts @@ -0,0 +1,214 @@ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { resolve as resolvePath } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseDocument } from 'yaml'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { testCapabilityDiscovery, type TestOptions } from '@adcp/sdk/testing'; +import { boundedDiscovery, checkProbe, MAX_ATTEMPTS, SmokeFailure } from '../../../scripts/external-smoke/check.js'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const url = 'https://smoke.example.com/mcp'; +const options: TestOptions = { auth: { type: 'bearer', token: 'test-only' } }; +type Result = Awaited>; +const success = (): Result => ({ + profile: { name: 'Test agent', tools: ['get_products'], adcp_version: 'v3', supported_protocols: ['media_buy'] }, + steps: [{ step: 'Discover agent capabilities', task: 'getAgentInfo', passed: true, duration_ms: 0 }], +}); +const failure = (error: string): Result => ({ + profile: { name: 'Unknown', tools: [] }, + steps: [{ step: 'Discover agent capabilities', task: 'getAgentInfo', passed: false, duration_ms: 0, error }], +}); +const resolved = { storyboards: ['test'], bundles: ['test'] } as unknown as ReturnType; + +afterEach(() => vi.restoreAllMocks()); + +describe('external smoke discovery', () => { + it('aborts a never-settling real SDK transport within budget, drains requests, and isolates the next attempt', async () => { + const unscopedFetch = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('Unscoped network request')); + let active = 0; + const signals: AbortSignal[] = []; + const fetchFn = vi.fn((_input, init) => new Promise((_resolve, reject) => { + active++; + const signal = init!.signal!; + signals.push(signal); + const abort = () => { active--; reject(signal.reason); }; + if (signal.aborted) abort(); + else signal.addEventListener('abort', abort, { once: true }); + })); + const attempts: AbortSignal[] = []; + const discover = vi.fn((endpoint, opts) => { + attempts.push(opts.signal!); + expect(opts.transport?.requestTimeoutMs).toBe(2000); + return testCapabilityDiscovery(endpoint, opts); + }); + const start = performance.now(); + for (let attempt = 0; attempt < 2; attempt++) { + await expect(boundedDiscovery(url, options, { + discover, fetchFn, attemptTimeoutMs: 250, requestTimeoutMs: 2000, + })).rejects.toMatchObject({ kind: 'transient' }); + expect(active).toBe(0); + } + expect(performance.now() - start).toBeLessThan(2000); + expect(fetchFn).toHaveBeenCalled(); + expect(unscopedFetch).not.toHaveBeenCalled(); + expect(signals.every(signal => signal.aborted)).toBe(true); + expect(attempts).toHaveLength(2); + expect(attempts[0]).not.toBe(attempts[1]); + const next = vi.fn(async (_endpoint, opts) => { + expect(opts.signal?.aborted).toBe(false); + expect(attempts).not.toContain(opts.signal); + return success(); + }); + await expect(boundedDiscovery(url, options, { discover: next })).resolves.toEqual(success()); + }); + + it('the real SDK consumes the explicit request timeout before the longer attempt budget', async () => { + let active = 0; + const fetchFn = vi.fn((_input, init) => new Promise((_resolve, reject) => { + active++; + const abort = () => { active--; reject(init!.signal!.reason); }; + if (init!.signal!.aborted) abort(); + else init!.signal!.addEventListener('abort', abort, { once: true }); + })); + const start = performance.now(); + const result = await boundedDiscovery(url, options, { fetchFn, attemptTimeoutMs: 2000, requestTimeoutMs: 50 }); + expect(result.steps.some(step => !step.passed)).toBe(true); + expect(performance.now() - start).toBeLessThan(1500); + expect(active).toBe(0); + await expect(checkProbe(url, options, {}, false, { discover: async () => result })) + .rejects.toMatchObject({ kind: 'transient' }); + }); + + it('bounds SDK calls that omit the signal and prevents late calls from starting another request', async () => { + let active = 0; + let lateFetch: typeof fetch; + const fetchFn = vi.fn((_input, init) => new Promise((_resolve, reject) => { + active++; + init!.signal!.addEventListener('abort', () => { active--; reject(init!.signal!.reason); }, { once: true }); + })); + const discover = vi.fn(async (_endpoint, opts) => { + lateFetch = opts.transport!.trustedFetchFn!; + await lateFetch(url); // Deliberately omit signal, like rc.35's second call. + return success(); + }); + await expect(boundedDiscovery(url, options, { discover, fetchFn, attemptTimeoutMs: 20 })).rejects.toMatchObject({ kind: 'transient' }); + expect(active).toBe(0); + await expect(lateFetch!(url)).rejects.toMatchObject({ kind: 'transient' }); + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + + it('classifies auth failure with a truthy empty profile using failed discovery evidence', async () => { + const discover = vi.fn().mockResolvedValue(failure('HTTP 401 Unauthorized')); + const resolve = vi.fn(); + await expect(checkProbe(url, {}, {}, true, { discover, resolve })).resolves.toBe('auth-required'); + expect(discover).toHaveBeenCalledTimes(1); + expect(resolve).not.toHaveBeenCalled(); + await expect(checkProbe(url, options, {}, false, { discover, resolve })).rejects.toMatchObject({ kind: 'auth' }); + expect(discover).toHaveBeenCalledTimes(2); + }); + + it.each(['missing step', 'failed step', 'skipped step', 'empty tools', 'missing version', 'empty versions'])( + 'rejects public discovery with %s', async condition => { + const result = success(); + if (condition === 'missing step') result.steps = []; + if (condition === 'failed step') result.steps[0].passed = false; + if (condition === 'skipped step') result.steps[0].skipped = true; + if (condition === 'empty tools') result.profile!.tools = []; + if (condition === 'missing version') delete result.profile!.adcp_version; + if (condition === 'empty versions') result.profile!.adcp_supported_versions = []; + const discover = vi.fn().mockResolvedValue(result); + await expect(checkProbe(url, {}, {}, true, { discover })).rejects.toThrow(); + expect(discover).toHaveBeenCalledTimes(1); + }, + ); + + it.each([false, true])('discovers exactly once before local resolver assertions (anonymous=%s)', async anonymous => { + const discover = vi.fn().mockResolvedValue(success()); + const resolve = vi.fn(() => { + expect(discover).toHaveBeenCalledExactlyOnceWith(url, anonymous ? {} : options); + return resolved; + }); + await expect(checkProbe(url, anonymous ? {} : options, {}, anonymous, { discover, resolve })) + .resolves.toBe(anonymous ? 'public' : 'authenticated'); + expect(resolve).toHaveBeenCalledTimes(1); + expect(resolve.mock.calls[0][0]).toMatchObject({ supported_protocols: ['media_buy'] }); + }); + + it.each(['ECONNRESET', 'HTTP 503 Service unavailable', 'Request timed out', 'EAI_AGAIN'])('retries %s exactly to the fixed bound', async error => { + const discover = vi.fn().mockResolvedValue(failure(error)); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toMatchObject({ kind: 'transient' }); + expect(discover).toHaveBeenCalledTimes(MAX_ATTEMPTS); + }); + + it('recovers on a fresh attempt after a thrown transient transport failure', async () => { + const discover = vi.fn().mockRejectedValueOnce(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } })).mockResolvedValueOnce(success()); + await expect(checkProbe(url, options, {}, false, { discover, resolve: () => resolved })).resolves.toBe('authenticated'); + expect(discover).toHaveBeenCalledTimes(2); + }); + + it.each(['schema validation failed: timeout field invalid', 'schema validation failed: HTTP 401', 'HTTP 400 invalid request', 'unknown network failure', 'HTTP 403 Forbidden', 'Expected 503 tools'])( + 'never retries %s', async error => { + const discover = vi.fn().mockResolvedValue(failure(error)); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toThrow(); + expect(discover).toHaveBeenCalledTimes(1); + }, + ); + + it('does not let a transient error hide a semantic validation failure', async () => { + const result = failure('HTTP 503'); + result.steps.push({ step: 'Validate capabilities', passed: false, duration_ms: 0 }); + const discover = vi.fn().mockResolvedValue(result); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toMatchObject({ kind: 'semantic' }); + expect(discover).toHaveBeenCalledTimes(1); + }); + + it('rejects profile schema issues even when discovery steps passed', async () => { + const result = success(); + result.profile!.capabilities_schema_issues = [{ pointer: '/adcp', message: 'invalid' }]; + const discover = vi.fn().mockResolvedValue(result); + await expect(checkProbe(url, options, {}, false, { discover })).rejects.toMatchObject({ kind: 'semantic' }); + expect(discover).toHaveBeenCalledTimes(1); + }); + + it.each(['storyboards', 'bundles', 'throw'])('does not retry local resolver failure: %s', async mode => { + const discover = vi.fn().mockResolvedValue(success()); + const resolve = vi.fn(() => { + if (mode === 'throw') throw new SmokeFailure('transient', 'HTTP 503'); + return { ...resolved, [mode]: [] }; + }); + await expect(checkProbe(url, {}, {}, true, { discover, resolve })).rejects.toThrow(); + expect(discover).toHaveBeenCalledTimes(1); + }); +}); + +describe('external smoke CI isolation', () => { + it('default integration selection cannot execute the external harness', () => { + const pkg = JSON.parse(readFileSync(resolvePath(root, 'package.json'), 'utf8')); + expect(pkg.scripts['test:server-integration']).toBe('vitest run server/tests/integration --config server/vitest.config.ts'); + const files = JSON.parse(execFileSync(process.execPath, [ + 'node_modules/vitest/vitest.mjs', 'list', 'server/tests/integration', + '--config', 'server/vitest.config.ts', '--filesOnly', '--json', + ], { cwd: root, encoding: 'utf8', timeout: 20_000 })).map((entry: { file: string }) => entry.file); + expect(files.length).toBeGreaterThan(0); + expect(files.every((file: string) => file.includes('/tests/integration/'))).toBe(true); + expect(files.some((file: string) => file.includes('external-smoke'))).toBe(false); + // Discovery alone cannot detect a later import of the live entry point. + for (const file of files) expect(readFileSync(file, 'utf8')).not.toMatch(/external-smoke|ADCP_SMOKE_CHECK/); + expect(pkg.scripts.test).not.toContain('test:external-smoke'); + }); + + it('the valid scheduled/manual workflow invokes the exact dedicated command', () => { + const document = parseDocument(readFileSync(resolvePath(root, '.github/workflows/external-smoke.yml'), 'utf8')); + expect(document.errors).toEqual([]); + const workflow = document.toJS(); + expect(Object.keys(workflow.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + expect(workflow.on.schedule).toHaveLength(1); + expect(workflow.permissions).toEqual({ contents: 'read' }); + expect(workflow.concurrency['cancel-in-progress']).toBe(false); + const pkg = JSON.parse(readFileSync(resolvePath(root, 'package.json'), 'utf8')); + expect(pkg.scripts['test:external-smoke']).toBe('tsx scripts/external-smoke/run.ts'); + expect(workflow.jobs.smoke.steps.filter((step: { run?: string }) => step.run === 'npm run test:external-smoke')).toHaveLength(1); + expect(workflow.jobs.smoke['timeout-minutes']).toBe(5); + }); +});