Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/workflows/external-smoke.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: External smoke

# Synthetic monitoring of the deployed public agent; not candidate validation
# or a release gate. Keep live requests out of required PR/push test shards.
on:
schedule:
- cron: '17 6 * * *'
workflow_dispatch: {}

permissions:
contents: read

concurrency:
group: external-smoke
cancel-in-progress: false

jobs:
smoke:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: '24'
cache: npm
- run: npm ci
env:
PUPPETEER_SKIP_DOWNLOAD: 'true'
- name: Check deployed public agent
timeout-minutes: 2
run: npm run test:external-smoke
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@
"test:redteam": "tsx server/src/addie/testing/redteam-cli.ts",
"test:server-unit": "vitest run --config server/vitest.config.ts tests/unit/",
"test:server-integration": "vitest run server/tests/integration --config server/vitest.config.ts",
"test:external-smoke": "tsx scripts/external-smoke/run.ts",
"test:openapi": "tsx scripts/generate-openapi.ts && git diff --exit-code static/openapi/registry.yaml || (echo 'OpenAPI spec is out of date. Run: npm run build:openapi' && exit 1)",
"test:registry": "vitest run server/tests --exclude 'server/tests/simulation/**'",
"test:simulation": "vitest run server/tests/simulation/scenarios --pool forks --poolOptions.forks.singleFork --testTimeout 120000",
Expand Down
25 changes: 25 additions & 0 deletions scripts/external-smoke/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# External smoke

`npm run test:external-smoke` checks the deployed public test agent using the
hosted compliance artifacts in this checkout. Override `ADCP_SMOKE_CHECK_URL`
and `ADCP_SMOKE_CHECK_TOKEN` for an operator-selected endpoint. This is synthetic
monitoring, not validation of candidate server code or a release gate. The
scheduled/manual `external-smoke.yml` workflow is its only CI caller.

Each authenticated/anonymous probe makes at most two discovery attempts, each
with a fresh AbortSignal and fetch closure, a 10s total
budget, and explicit 4s SDK transport timeout. Only categorized transient
transport failures retry; schema, assertion, auth and unknown errors do not.
Authenticated discovery runs once per attempt and its profile is reused for
local storyboard resolution. Anonymous success meets the same checks, or
failed discovery must supply concrete authentication error evidence.

SDK 14.0.0-rc.35 does not pass the signal to its second capability call. The
local `trustedFetchFn` composes the attempt signal with the transport signal
and rejects new requests after abort, including late SDK calls. Native fetch
aborts in-flight requests and response bodies. A deadline race bounds the
caller, but cannot cancel SDK JavaScript or a custom fetch implementation that
ignores abort; those promises may continue unwinding. SDK connection cache
entries are isolated by fetch identity and may remain until the runner exits.
This test-only trusted fetch uses Node's networking for
operator-selected endpoints; it is not a hosted untrusted-URL security boundary.
148 changes: 148 additions & 0 deletions scripts/external-smoke/check.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
import assert from 'node:assert/strict';
import {
testCapabilityDiscovery,
resolveStoryboardsForCapabilities,
type TestOptions,
} from '@adcp/sdk/testing';

export const MAX_ATTEMPTS = 2;
export const ATTEMPT_TIMEOUT_MS = 10_000;
export const REQUEST_TIMEOUT_MS = 4_000;

type Discovery = typeof testCapabilityDiscovery;
type DiscoveryResult = Awaited<ReturnType<Discovery>>;
type Resolver = typeof resolveStoryboardsForCapabilities;
type FailureKind = 'auth' | 'transient' | 'semantic';

export class SmokeFailure extends Error {
constructor(readonly kind: FailureKind, message: string) {
super(message);
this.name = 'SmokeFailure';
}
}

// SDK step errors are strings. Keep this allowlist narrow; unknown errors fail
// once. In particular, a generic mention of "auth" or "network" proves nothing.
function failureKind(error: unknown): FailureKind {
if (error instanceof SmokeFailure) return error.kind;
const value = error as { name?: string; code?: string; status?: number; message?: string; cause?: unknown } | null;
const evidence = typeof error === 'string' ? error :
[value?.name, value?.code, value?.status, value?.message].join(' ');
if (/\b(?:AssertionError|ZodError|SchemaValidationError)\b|schema|validation|unsupported version/i.test(evidence)) return 'semantic';
if ([401, 403].includes(value?.status ?? 0) || /\b(?:unauthorized|UnauthorizedError|forbidden|AUTH_REQUIRED|AUTHENTICATION_REQUIRED|NeedsAuthorizationError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:401|403)\b|authentication required|invalid (?:bearer )?token/i.test(evidence)) return 'auth';
if ([408, 429, 502, 503, 504].includes(value?.status ?? 0) || /\b(?:ETIMEDOUT|ECONNRESET|ECONNREFUSED|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT|UND_ERR_SOCKET|TimeoutError|SmokeTransientTransportError)\b|\b(?:HTTP|status(?: code)?)[\s:=]+(?:408|429|502|503|504)\b|request timed out|request timeout|version negotiation probe timed out/i.test(evidence)) return 'transient';
return value?.cause ? failureKind(value.cause) : 'semantic';
}

/** Abort the actual fetch boundary, including SDK calls that omit options.signal. */
export async function boundedDiscovery(
url: string,
options: TestOptions,
{
discover = testCapabilityDiscovery,
fetchFn = fetch,
attemptTimeoutMs = ATTEMPT_TIMEOUT_MS,
requestTimeoutMs = REQUEST_TIMEOUT_MS,
}: { discover?: Discovery; fetchFn?: typeof fetch; attemptTimeoutMs?: number; requestTimeoutMs?: number } = {},
): Promise<DiscoveryResult> {
const controller = new AbortController();
const timeout = new SmokeFailure('transient', `Discovery attempt timed out after ${attemptTimeoutMs}ms`);
const timer = setTimeout(() => controller.abort(timeout), attemptTimeoutMs);
const scopedFetch: typeof fetch = async (input, init) => {
controller.signal.throwIfAborted();
const signals = [controller.signal, init?.signal, input instanceof Request ? input.signal : undefined]
.filter((signal): signal is AbortSignal => signal != null);
try {
return await fetchFn(input, { ...init, signal: AbortSignal.any(signals) });
} catch (error) {
// Preserve categorized cause evidence before rc.35 reduces it to a string.
const kind = failureKind(error);
if (kind === 'transient') throw new SmokeFailure(kind, `SmokeTransientTransportError: ${String(error)}`);
throw error;
}
};
let onAbort: () => void = () => {};
const aborted = new Promise<never>((_, reject) => {
onAbort = () => reject(controller.signal.reason);
controller.signal.addEventListener('abort', onAbort, { once: true });
});
try {
// rc.35 keys connections by fetch identity, so attempts cannot reuse them.
return await Promise.race([
discover(url, {
...options,
signal: controller.signal,
transport: { ...options.transport, requestTimeoutMs, trustedFetchFn: scopedFetch },
}),
aborted,
]);
} finally {
clearTimeout(timer);
controller.signal.removeEventListener('abort', onAbort);
// Also close pending response bodies / late SDK calls after success or error.
// SDK promises can still unwind; this does not claim to cancel JavaScript.
controller.abort(new Error('Smoke attempt finished'));
}
}

function assertDiscovery(result: DiscoveryResult): void {
const schemaIssues = result.profile?.capabilities_schema_issues;
if (schemaIssues?.length) throw new SmokeFailure('semantic', `Capability schema errors: ${JSON.stringify(schemaIssues)}`);
const failures = result.steps.filter(step => !step.passed && !step.skipped).map(step => ({
kind: step.task === 'getAgentInfo' || step.task === 'get_adcp_capabilities'
? failureKind(step.error) : 'semantic' as FailureKind,
message: `${step.step}: ${step.error ?? step.details ?? 'failed'}`,
}));
if (result.profile?.capabilities_probe_error) failures.push({
kind: failureKind(result.profile.capabilities_probe_error),
message: result.profile.capabilities_probe_error,
});
if (failures.length) {
// Mixed semantic/auth + transient failures must never turn into a retry.
const kind = failures.some(f => f.kind === 'semantic') ? 'semantic' :
failures.some(f => f.kind === 'auth') ? 'auth' : 'transient';
throw new SmokeFailure(kind, failures.map(f => f.message).join('; '));
}
assert(result.steps.some(step => step.task === 'getAgentInfo' && step.passed && !step.skipped), 'No passed discovery step');
assert(result.profile?.tools.length, 'Discovery returned no tools');
const versions = result.profile.adcp_supported_versions;
assert(versions === undefined
? ['v2', 'v3'].includes(result.profile.adcp_version ?? '')
: versions.length > 0 && versions.every(version => /^\d+\.\d+(?:\.\d+)?(?:-[\w.-]+)?$/.test(version)),
'Discovery returned no supported version');
}

export async function checkProbe(
url: string,
options: TestOptions,
resolverOptions: Parameters<Resolver>[1],
anonymous: boolean,
{
discover = (endpoint, opts) => boundedDiscovery(endpoint, opts),
resolve = resolveStoryboardsForCapabilities,
}: { discover?: Discovery; resolve?: Resolver } = {},
): Promise<'public' | 'authenticated' | 'auth-required'> {
for (let attempt = 1; ; attempt++) {
let result: DiscoveryResult;
try {
// Exactly one authenticated discovery per attempt; reuse its profile.
result = await discover(url, options);
assertDiscovery(result);
} catch (error) {
const kind = failureKind(error);
if (anonymous && kind === 'auth') return 'auth-required';
if (kind !== 'transient' || attempt === MAX_ATTEMPTS) throw error;
continue;
}
const profile = result.profile!;
const resolved = resolve({
supported_protocols: profile.supported_protocols ?? [],
specialisms: profile.specialisms ?? [],
major_versions: profile.adcp_major_versions,
supported_versions: profile.adcp_supported_versions,
}, resolverOptions);
assert(resolved.storyboards.length > 0, 'No applicable storyboards');
assert(resolved.bundles.length > 0, 'No applicable bundles');
return anonymous ? 'public' : 'authenticated';
}
}
17 changes: 17 additions & 0 deletions scripts/external-smoke/run.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { PUBLIC_TEST_AGENT } from '../../server/src/config/test-agent.js';
import {
hostedComplianceTarget,
hostedComplianceOptions,
withHostedTestOptions,
} from '../../server/src/services/hosted-compliance-version.js';
import { checkProbe } from './check.js';

const url = process.env.ADCP_SMOKE_CHECK_URL || PUBLIC_TEST_AGENT.url;
const token = process.env.ADCP_SMOKE_CHECK_TOKEN || PUBLIC_TEST_AGENT.token;
const target = hostedComplianceTarget();
const resolverOptions = hostedComplianceOptions(target);

await checkProbe(url, withHostedTestOptions({ auth: { type: 'bearer', token } }, target), resolverOptions, false);
console.log('Authenticated discovery and storyboard resolution passed.');
const anonymous = await checkProbe(url, withHostedTestOptions({}, target), resolverOptions, true);
console.log(`Anonymous discovery: ${anonymous}.`);
81 changes: 0 additions & 81 deletions server/tests/integration/external-smoke-check.test.ts

This file was deleted.

Loading
Loading