Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions scripts/external-smoke/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,25 @@ Each authenticated/anonymous probe makes at most two discovery attempts, each
with a fresh AbortSignal and fetch closure, a 10s total
budget, and explicit 4s SDK transport timeout. Only categorized transient
transport failures retry; schema, assertion, auth and unknown errors do not.
Authenticated discovery runs once per attempt and its profile is reused for
local storyboard resolution. Anonymous success meets the same checks, or
failed discovery must supply concrete authentication error evidence.
Authenticated discovery runs once per negotiated release per attempt, and its
profile is reused for local storyboard resolution. Anonymous success meets the
same checks, or failed discovery must supply concrete authentication error
evidence (including the SDK's OAuth challenge, "requires OAuth authorization").

The probe pins the default hosted line (3.0). A 3.0 pin cannot carry
`adcp_version`, so the SDK sends only `adcp_major_version: 3`, and the agent
may serve and echo its highest 3.x release. `@adcp/sdk` validates the response
against the 3.0 pin regardless. The versioning spec says buyers validate
against the echoed release. When the echo differs from the pin, the smoke check
therefore requires the echo to be listed in `adcp.supported_versions`,
discovers again pinned to that release, validates strictly against it, and
resolves storyboards from that release's hosted bundle. An echo with no hosted
bundle, a different major, or a different release on the pinned probe fails.

SDK 15.8.1 also keys `comply_test_controller` as protocol `compliance` in its
tool cross-check, while it parses the `compliance_testing` block as
`compliance_testing`. The check waives only that exact cross-check message,
and only when the response carries the `compliance_testing` block.

SDK 14.0.0-rc.35 does not pass the signal to its second capability call. The
local `trustedFetchFn` composes the attempt signal with the transport signal
Expand Down
107 changes: 100 additions & 7 deletions scripts/external-smoke/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,16 @@ export class SmokeFailure extends Error {

// SDK step errors are strings. Keep this allowlist narrow; unknown errors fail
// once. In particular, a generic mention of "auth" or "network" proves nothing.
// `requires OAuth authorization` is the message of the SDK's
// NeedsAuthorizationError (a 401 Bearer challenge), which reaches us only as a
// step string.
function failureKind(error: unknown): FailureKind {
if (error instanceof SmokeFailure) return error.kind;
const value = error as { name?: string; code?: string; status?: number; message?: string; cause?: unknown } | null;
const evidence = typeof error === 'string' ? error :
[value?.name, value?.code, value?.status, value?.message].join(' ');
if (/\b(?:AssertionError|ZodError|SchemaValidationError)\b|schema|validation|unsupported version/i.test(evidence)) return 'semantic';
if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token/i.test(evidence)) return 'auth';
if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token|\brequires OAuth authorization\b/i.test(evidence)) return 'auth';
if ([408, 429, 502, 503, 504].includes(value?.status ?? 0) || /\b(?:ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET|TimeoutError|SmokeTransientTransportError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:408|429|502|503|504)\b|request timed out|request timeout|version negotiation probe timed out/i.test(evidence)) return 'transient';
return value?.cause ? failureKind(value.cause) : 'semantic';
}
Expand Down Expand Up @@ -85,10 +88,29 @@ export async function boundedDiscovery(
}
}

// @adcp/sdk 15.8.1 (and 15.9.1) parses a `compliance_testing` capability
// block as protocol `compliance_testing`, but its tool cross-check keys
// comply_test_controller under `compliance`. Every agent that correctly
// declares the block therefore fails the step with exactly this message.
// Waive only that exact message, and only when the agent's response carries
// the block (the SDK also adds the protocol when the block is missing). Any
// other cross-check issue still fails.
const SDK_COMPLIANCE_CROSS_CHECK_STEP = 'Cross-validate protocols and tools';
const SDK_COMPLIANCE_CROSS_CHECK_ERROR = 'Tools suggest unreported protocols: compliance (has: [comply_test_controller])';

function isSdkComplianceCrossCheckFalsePositive(step: DiscoveryResult['steps'][number], result: DiscoveryResult): boolean {
return step.step === SDK_COMPLIANCE_CROSS_CHECK_STEP
&& step.error === SDK_COMPLIANCE_CROSS_CHECK_ERROR
&& result.profile?.supported_protocols?.includes('compliance_testing') === true
&& typeof (result.profile.raw_capabilities as { compliance_testing?: unknown } | undefined)?.compliance_testing === 'object';
}

function assertDiscovery(result: DiscoveryResult): void {
const schemaIssues = result.profile?.capabilities_schema_issues;
if (schemaIssues?.length) throw new SmokeFailure('semantic', `Capability schema errors: ${JSON.stringify(schemaIssues)}`);
const failures = result.steps.filter(step => !step.passed && !step.skipped).map(step => ({
const failed = result.steps.filter(step => !step.passed && !step.skipped
&& !isSdkComplianceCrossCheckFalsePositive(step, result));
const failures = failed.map(step => ({
kind: step.task === 'getAgentInfo' || step.task === 'get_adcp_capabilities'
? failureKind(step.error) : 'semantic' as FailureKind,
message: `${step.step}: ${step.error ?? step.details ?? 'failed'}`,
Expand All @@ -112,21 +134,92 @@ function assertDiscovery(result: DiscoveryResult): void {
'Discovery returned no supported version');
}

type ResolverOptions = Parameters<Resolver>[1];

/** Probe and storyboard-resolution options pinned to one release. */
export interface ReleaseProbe {
options: TestOptions;
resolverOptions: ResolverOptions;
}

/** Builds a probe pinned to the release the agent said it served. */
export type RepinProbe = (release: string) => ReleaseProbe;

const RELEASE_RE = /^([1-9]\d*)\.(\d+)(?:\.\d+)?(?:-([0-9A-Za-z.-]+))?$/;

/** Release-precision wire value (`3.0`, `3.2-rc.7`) for a pin or an echo. */
export function releasePrecision(version: unknown): string | undefined {
const match = typeof version === 'string' ? version.match(RELEASE_RE) : null;
return match ? `${match[1]}.${match[2]}${match[3] ? `-${match[3]}` : ''}` : undefined;
}

/** The release the agent echoed on `get_adcp_capabilities`, if any. */
function servedRelease(result: DiscoveryResult): string | undefined {
const raw = result.profile?.raw_capabilities as { adcp_version?: unknown } | undefined;
return releasePrecision(raw?.adcp_version);
}

/**
* The SDK validates capabilities against the caller's pin, not the served
* release. A 3.0 pin cannot carry `adcp_version`, so the SDK sends only
* `adcp_major_version: 3` and the agent may serve its highest 3.x release
* (docs/reference/versioning.mdx, "Server resolves"). The spec says buyers
* validate against the echoed release, not their pin ("Server echoes").
*
* When the echo differs from the pin, the agent must advertise the echoed
* release. The probe then discovers again pinned to it, so the SDK validates
* strictly against that release, and resolves storyboards from that
* release's bundle. The pinned probe must keep serving the same release.
*/
async function negotiateServedRelease(
url: string,
probe: ReleaseProbe,
result: DiscoveryResult,
discover: Discovery,
repin: RepinProbe | undefined,
): Promise<{ probe: ReleaseProbe; result: DiscoveryResult }> {
const served = servedRelease(result);
const pinned = releasePrecision(probe.options.wireAdcpVersion ?? probe.options.adcpVersion);
if (!served || !pinned || served === pinned || !repin) return { probe, result };
const issues = result.profile?.capabilities_schema_issues;
const context = issues?.length ? `; issues against the ${pinned} pin: ${JSON.stringify(issues)}` : '';
if (served.split('.')[0] !== pinned.split('.')[0]) {
throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin; majors must agree${context}`);
}
if (!result.profile?.adcp_supported_versions?.includes(served)) {
throw new SmokeFailure('semantic', `Agent served AdCP ${served}, which its supported_versions do not list${context}`);
}
let servedProbe: ReleaseProbe;
try {
servedProbe = repin(served);
} catch (error) {
throw new SmokeFailure('semantic', `Agent served AdCP ${served}, but no hosted bundle is available to validate it: ${error instanceof Error ? error.message : String(error)}${context}`);
}
const servedResult = await discover(url, servedProbe.options);
const reserved = servedRelease(servedResult);
if (reserved !== undefined && reserved !== served) {
throw new SmokeFailure('semantic', `Agent served AdCP ${served} to a ${pinned} pin, but ${reserved} when pinned to ${served}`);
}
return { probe: servedProbe, result: servedResult };
}

export async function checkProbe(
url: string,
options: TestOptions,
resolverOptions: Parameters<Resolver>[1],
resolverOptions: ResolverOptions,
anonymous: boolean,
{
discover = (endpoint, opts) => boundedDiscovery(endpoint, opts),
resolve = resolveStoryboardsForCapabilities,
}: { discover?: Discovery; resolve?: Resolver } = {},
repin,
}: { discover?: Discovery; resolve?: Resolver; repin?: RepinProbe } = {},
): Promise<'public' | 'authenticated' | 'auth-required'> {
for (let attempt = 1; ; attempt++) {
let result: DiscoveryResult;
let probe: ReleaseProbe = { options, resolverOptions };
try {
// Exactly one authenticated discovery per attempt; reuse its profile.
result = await discover(url, options);
// One discovery per negotiated release per attempt; reuse its profile.
({ probe, result } = await negotiateServedRelease(url, probe, await discover(url, options), discover, repin));
assertDiscovery(result);
} catch (error) {
const kind = failureKind(error);
Expand All @@ -140,7 +233,7 @@ export async function checkProbe(
specialisms: profile.specialisms ?? [],
major_versions: profile.adcp_major_versions,
supported_versions: profile.adcp_supported_versions,
}, resolverOptions);
}, probe.resolverOptions);
assert(resolved.storyboards.length > 0, 'No applicable storyboards');
assert(resolved.bundles.length > 0, 'No applicable bundles');
return anonymous ? 'public' : 'authenticated';
Expand Down
18 changes: 15 additions & 3 deletions scripts/external-smoke/run.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,29 @@
import type { TestOptions } from '@adcp/sdk/testing';
import { PUBLIC_TEST_AGENT } from '../../server/src/config/test-agent.js';
import {
hostedComplianceTarget,
hostedComplianceOptions,
withHostedTestOptions,
} from '../../server/src/services/hosted-compliance-version.js';
import { checkProbe } from './check.js';
import { checkProbe, type RepinProbe } from './check.js';

const url = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url;
const token = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token;
const target = hostedComplianceTarget();
const resolverOptions = hostedComplianceOptions(target);
// When the agent serves a different release than the default pin, validate
// and resolve against this checkout's hosted bundle for the served release.
const repinFor = (base: Partial<TestOptions>): RepinProbe => release => {
const served = hostedComplianceTarget(release);
return { options: withHostedTestOptions(base, served), resolverOptions: hostedComplianceOptions(served) };
};

await checkProbe(url, withHostedTestOptions({ auth: { type: 'bearer', token } }, target), resolverOptions, false);
const authenticated: Partial<TestOptions> = { auth: { type: 'bearer', token } };
await checkProbe(url, withHostedTestOptions(authenticated, target), resolverOptions, false, {
repin: repinFor(authenticated),
});
console.log('Authenticated discovery and storyboard resolution passed.');
const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true);
const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true, {
repin: repinFor({}),
});
console.log(`Anonymous discovery: ${anonymous}.`);
Loading
Loading