Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 28 additions & 10 deletions server/src/capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { AAO_UA_DISCOVERY } from "./config/user-agents.js";
import { logOutboundRequest } from "./db/outbound-log-db.js";
import { agentConfigAuthFields, type SdkAuth } from "./services/sdk-auth-adapter.js";
import { withSdkSafeTransport } from "./utils/sdk-safe-fetch.js";
import { isComplianceRefreshAccessFailure } from "./services/compliance-refresh-authorization.js";
import Ajv, { type ValidateFunction } from "ajv";
import addFormats from "ajv-formats";
import { readFileSync } from "node:fs";
Expand Down Expand Up @@ -473,7 +474,7 @@ export class CapabilityDiscovery {

constructor() {}

async discoverCapabilities(agent: Agent, auth?: SdkAuth, forceRefresh = false): Promise<AgentCapabilityProfile> {
async discoverCapabilities(agent: Agent, auth?: SdkAuth, forceRefresh = false, checkpoint?: () => Promise<void>): Promise<AgentCapabilityProfile> {
// Skip cache when auth is provided — manual owner-triggered refresh
// wants fresh data and may previously have cached an unauthed
// discovery_error result. Periodic crawls (no auth) keep the cache.
Expand All @@ -491,7 +492,8 @@ export class CapabilityDiscovery {
const startTime = Date.now();
try {
const protocol = agent.protocol || "mcp";
const tools = await this.discoverTools(agent.url, protocol, auth);
const tools = await this.discoverTools(agent.url, protocol, auth, checkpoint);
await checkpoint?.();

logOutboundRequest({
agent_url: agent.url,
Expand Down Expand Up @@ -520,11 +522,11 @@ export class CapabilityDiscovery {
// deadline can cause the probe to time out on a healthy but slow endpoint.
const hasGetAdcpCaps = toolNames.has('get_adcp_capabilities');
const rawAdcpCaps: Record<string, unknown> | undefined = hasGetAdcpCaps
? await this.fetchRawAdcpCapabilities(agent, auth)
? await this.fetchRawAdcpCapabilities(agent, auth, checkpoint)
: undefined;

if (CapabilityDiscovery.CREATIVE_TOOLS.some(t => toolNames.has(t))) {
const creativeResult = await this.analyzeCreativeCapabilities(agent, tools, auth, rawAdcpCaps);
const creativeResult = await this.analyzeCreativeCapabilities(agent, tools, auth, rawAdcpCaps, checkpoint);
profile.creative_capabilities = creativeResult.capabilities;
if (creativeResult.probeFailed) profile.creative_capabilities_probe_failed = true;
}
Expand All @@ -548,9 +550,12 @@ export class CapabilityDiscovery {
// tool list an agent advertises behind auth may differ from the
// public-facing one, and the cache is read by unauthed periodic
// crawls + the public registry render.
await checkpoint?.();
if (!auth) this.cache.set(agent.url, profile);
return profile;
} catch (error: any) {
if (isComplianceRefreshAccessFailure(error)) throw error;
await checkpoint?.();
logOutboundRequest({
agent_url: agent.url,
request_type: 'discovery',
Expand Down Expand Up @@ -578,15 +583,15 @@ export class CapabilityDiscovery {
}
}

private async discoverTools(url: string, protocol: "mcp" | "a2a", auth?: SdkAuth): Promise<ToolCapability[]> {
private async discoverTools(url: string, protocol: "mcp" | "a2a", auth?: SdkAuth, checkpoint?: () => Promise<void>): Promise<ToolCapability[]> {
if (protocol === "a2a") {
return this.discoverA2ATools(url, auth);
return this.discoverA2ATools(url, auth, checkpoint);
} else {
return this.discoverMCPTools(url, auth);
return this.discoverMCPTools(url, auth, checkpoint);
}
}

private async discoverMCPTools(url: string, auth?: SdkAuth): Promise<ToolCapability[]> {
private async discoverMCPTools(url: string, auth?: SdkAuth, checkpoint?: () => Promise<void>): Promise<ToolCapability[]> {
try {
// Use AdCPClient to connect to agent
const { AdCPClient } = await import("@adcp/sdk");
Expand All @@ -606,6 +611,7 @@ export class CapabilityDiscovery {
}));
const client = multiClient.agent("discovery");

await checkpoint?.();
const agentInfo = await client.getAgentInfo();
logger.debug({ url, toolCount: agentInfo.tools.length }, 'MCP discovery completed');

Expand All @@ -616,6 +622,7 @@ export class CapabilityDiscovery {
verified_at: new Date().toISOString(),
}));
} catch (error: any) {
if (isComplianceRefreshAccessFailure(error)) throw error;
// Re-throw AuthenticationRequiredError to preserve OAuth metadata for callers
if (error instanceof AuthenticationRequiredError) {
logger.info({ url, hasOAuth: error.hasOAuth }, 'MCP agent requires OAuth authentication');
Expand All @@ -633,7 +640,7 @@ export class CapabilityDiscovery {
}
}

private async discoverA2ATools(url: string, auth?: SdkAuth): Promise<ToolCapability[]> {
private async discoverA2ATools(url: string, auth?: SdkAuth, checkpoint?: () => Promise<void>): Promise<ToolCapability[]> {
try {
// Use AdCPClient to connect to agent
const { AdCPClient } = await import("@adcp/sdk");
Expand All @@ -650,6 +657,7 @@ export class CapabilityDiscovery {
}));
const client = multiClient.agent("discovery");

await checkpoint?.();
const agentInfo = await client.getAgentInfo();
logger.debug({ url, toolCount: agentInfo.tools.length }, 'A2A discovery completed');

Expand All @@ -660,6 +668,7 @@ export class CapabilityDiscovery {
verified_at: new Date().toISOString(),
}));
} catch (error: any) {
if (isComplianceRefreshAccessFailure(error)) throw error;
// Re-throw AuthenticationRequiredError to preserve OAuth metadata for callers
if (error instanceof AuthenticationRequiredError) {
logger.info({ url, hasOAuth: error.hasOAuth }, 'A2A agent requires OAuth authentication');
Expand Down Expand Up @@ -726,6 +735,7 @@ export class CapabilityDiscovery {
tools: ToolCapability[],
auth?: SdkAuth,
rawAdcpCaps?: Record<string, unknown>,
checkpoint?: () => Promise<void>,
): Promise<{ capabilities: CreativeCapabilities; probeFailed: boolean }> {
const toolNames = new Set(tools.map((t) => t.name.toLowerCase()));
let declared: { ok: true; capabilities?: CreativeCapabilities } | { ok: false };
Expand All @@ -743,7 +753,7 @@ export class CapabilityDiscovery {
}
}
} else {
declared = await this.fetchCreativeCapabilities(agent, auth);
declared = await this.fetchCreativeCapabilities(agent, auth, checkpoint);
}
} else {
declared = { ok: true, capabilities: undefined };
Expand Down Expand Up @@ -774,6 +784,7 @@ export class CapabilityDiscovery {
private async fetchCreativeCapabilities(
agent: Agent,
auth?: SdkAuth,
checkpoint?: () => Promise<void>,
): Promise<{ ok: true; capabilities?: CreativeCapabilities } | { ok: false }> {
try {
const { AdCPClient } = await import("@adcp/sdk");
Expand All @@ -788,7 +799,9 @@ export class CapabilityDiscovery {
transport: { maxResponseBytes: 1024 * 1024 },
}));
const client = multiClient.agent("discovery");
await checkpoint?.();
const result = await client.getAdcpCapabilities({}, undefined, { timeout: 10_000 });
await checkpoint?.();
if (!result?.success) {
logger.debug({ url: agent.url, error: result?.error }, 'Creative capability probe returned an error result');
return { ok: false };
Expand All @@ -797,6 +810,7 @@ export class CapabilityDiscovery {
if (creative === undefined || creative === null) return { ok: true };
return { ok: true, capabilities: await sanitizeCreativeCapabilities(creative) };
} catch (err: any) {
if (isComplianceRefreshAccessFailure(err)) throw err;
logger.debug({ url: agent.url, err: err?.message }, 'Creative capability fetch failed');
return { ok: false };
}
Expand All @@ -816,6 +830,7 @@ export class CapabilityDiscovery {
private async fetchRawAdcpCapabilities(
agent: Agent,
auth?: SdkAuth,
checkpoint?: () => Promise<void>,
): Promise<Record<string, unknown> | undefined> {
try {
const { AdCPClient } = await import("@adcp/sdk");
Expand All @@ -830,10 +845,13 @@ export class CapabilityDiscovery {
transport: { maxResponseBytes: 1024 * 1024 },
}));
const client = multiClient.agent("discovery");
await checkpoint?.();
const result = await client.getAdcpCapabilities({}, undefined, { timeout: 10_000 });
await checkpoint?.();
if (!result?.success) return undefined;
return result.data as Record<string, unknown> | undefined;
} catch (err: any) {
if (isComplianceRefreshAccessFailure(err)) throw err;
logger.debug({ url: agent.url, err: err?.message }, 'get_adcp_capabilities fetch failed');
return undefined;
}
Expand Down
121 changes: 77 additions & 44 deletions server/src/crawler.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { Agent, FederatedAgent, ResolvedBrand } from "./types.js";
import type { Client } from "pg";
import type { Client, PoolClient } from "pg";
import { PropertyCrawler, getPropertyIndex, type AgentInfo, type CrawlResult } from "@adcp/sdk";
import { sanitizeAdagentsProperty } from "./discovery/property-index-guard.js";
import { FederatedIndexService } from "./federated-index.js";
Expand Down Expand Up @@ -34,12 +34,13 @@ import {
type AgentInventoryProfilesDatabase,
type ProfileUpsertInput,
} from "./db/agent-inventory-profiles-db.js";
import { getDedicatedClient, query, withDatabaseDeadline } from "./db/client.js";
import { getClient, getDedicatedClient, query, withDatabaseDeadline } from "./db/client.js";
import { PropertyDatabase } from "./db/property-db.js";
import { verifyHostedPropertyOrigin } from "./services/hosted-property-origin-verifier.js";
import { insertTypeReclassification } from "./db/type-reclassification-log-db.js";
import { resolveUserAgentAuth } from "./routes/helpers/resolve-user-agent-auth.js";
import { adaptAuthForSdk, type SdkAuth } from "./services/sdk-auth-adapter.js";
import { isComplianceRefreshAccessFailure, type ComplianceRefreshAuthorizationGuard } from "./services/compliance-refresh-authorization.js";
import {
PublisherCrawlRequestsDatabase,
type CreatePublisherCrawlRequestInput,
Expand Down Expand Up @@ -1747,7 +1748,11 @@ export class CrawlerService {
* route handler maps that to a 502 so the user sees why the refresh
* couldn't happen (timeout, DNS, OAuth wall, etc).
*/
async refreshSingleAgent(agentUrl: string, options: { auth?: SdkAuth; ownerOrgId?: string } = {}): Promise<{
async refreshSingleAgent(agentUrl: string, options: {
auth?: SdkAuth;
ownerOrgId?: string;
authorization?: ComplianceRefreshAuthorizationGuard;
} = {}): Promise<{
online: boolean;
tools_count: number | null;
response_time_ms: number | null;
Expand All @@ -1758,7 +1763,7 @@ export class CrawlerService {
error?: string;
}> {
const PROBE_TIMEOUT_MS = 10000;
const { auth, ownerOrgId } = options;
const { auth, ownerOrgId, authorization } = options;

const pausedUrls = await this.getPausedAgentUrls();
if (pausedUrls.has(agentUrl)) {
Expand Down Expand Up @@ -1792,80 +1797,108 @@ export class CrawlerService {
// with no saved owner auth would still read the shared 15-minute
// capability/health cache and keep reporting a fixed-then-redeployed
// issue as unresolved (#5777).
await authorization?.checkpoint();
const profile = await Promise.race([
this.capabilityDiscovery.discoverCapabilities(agent, auth, true),
this.capabilityDiscovery.discoverCapabilities(agent, auth, true, authorization?.checkpoint),
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error('Probe timeout')), PROBE_TIMEOUT_MS)
),
]);
await authorization?.checkpoint();
if (authorization && profile.agent_url !== agentUrl) {
throw Object.assign(new Error('Capability profile did not match the refresh target'), { code: 'probe_failed' });
}

const inferredType = this.capabilityDiscovery.inferTypeFromProfile(profile);
const effectiveType = knownType || inferredType;
const agentForHealth: Agent = { ...agent, type: effectiveType as Agent['type'], protocol: profile.protocol };

const [health, stats] = await Promise.all([
Promise.race([
this.healthChecker.checkHealth(agentForHealth, auth, true),
this.healthChecker.checkHealth(agentForHealth, auth, true, authorization?.checkpoint),
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error('Health timeout')), PROBE_TIMEOUT_MS)
),
]).catch((err): import('./types.js').AgentHealth => ({
online: false,
checked_at: new Date().toISOString(),
error: err instanceof Error ? err.message : 'health check failed',
})),
]).catch((err): import('./types.js').AgentHealth => {
if (isComplianceRefreshAccessFailure(err)) throw err;
return {
online: false,
checked_at: new Date().toISOString(),
error: err instanceof Error ? err.message : 'health check failed',
};
}),
Promise.race([
this.healthChecker.getStats(agentForHealth, auth, true),
this.healthChecker.getStats(agentForHealth, auth, true, authorization?.checkpoint),
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error('Stats timeout')), PROBE_TIMEOUT_MS)
),
]).catch((): import('./types.js').AgentStats => ({})),
]);

await Promise.all([
this.snapshotDb.upsertCapabilities(
profile,
inferredType === 'unknown' ? null : inferredType,
{ trackUnknownProbe: !knownType },
),
this.snapshotDb.upsertHealth(agentUrl, health, stats),
]).catch((err): import('./types.js').AgentStats => {
if (isComplianceRefreshAccessFailure(err)) throw err;
return {};
}),
]);
await authorization?.checkpoint();

// Same type-promotion policy as refreshAgentSnapshots: promote when
// stored is unknown; log disagreement without auto-flipping (see #3538).
const canPromote = inferredType !== 'unknown' && !knownType;
const isDisagreement =
!!knownType && inferredType !== 'unknown' && knownType !== inferredType;

if (isDisagreement) {
log.warn(
{ url: agentUrl, knownType, inferredType },
'Agent type disagreement: stored vs probed. Run backfill to reconcile.'
);
await insertTypeReclassification({
agentUrl,
oldType: knownType ?? null,
newType: inferredType,
source: 'crawler_promote',
notes: { decision: 'logged_only_no_promote', triggered_by: 'manual_refresh' },
});
}

let typePromoted = false;
if (canPromote) {
await this.federatedIndex.updateAgentMetadata(agentUrl, {
agent_type: inferredType,
protocol: profile.protocol,
});
typePromoted = true;
const writeSnapshotsAndType = async (client?: PoolClient) => {
await Promise.all([
this.snapshotDb.upsertCapabilities(
profile,
inferredType === 'unknown' ? null : inferredType,
{ trackUnknownProbe: !knownType, ...(client ? { client } : {}) },
),
this.snapshotDb.upsertHealth(agentUrl, health, stats, client),
]);
if (isDisagreement) {
log.warn(
{ url: agentUrl, knownType, inferredType },
'Agent type disagreement: stored vs probed. Run backfill to reconcile.'
);
await insertTypeReclassification({
agentUrl,
oldType: knownType ?? null,
newType: inferredType,
source: 'crawler_promote',
notes: { decision: 'logged_only_no_promote', triggered_by: 'manual_refresh' },
}, client);
}
if (canPromote) {
await this.federatedIndex.updateAgentMetadata(agentUrl, {
agent_type: inferredType,
protocol: profile.protocol,
}, client);
}
};
if (authorization) {
// Keep authorization locks and every mutation on the same connection.
// Losing this transaction cannot leave a detached writer committing.
const client = await getClient();
try {
await client.query('BEGIN');
await authorization.beforeWrite(client, agentUrl);
await writeSnapshotsAndType(client);
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK').catch(() => undefined);
throw error;
} finally {
client.release();
}
} else {
await writeSnapshotsAndType();
}

return {
online: health.online,
tools_count: health.tools_count ?? null,
response_time_ms: health.response_time_ms ?? null,
inferred_type: inferredType,
type_promoted: typePromoted,
type_promoted: canPromote,
oauth_required: profile.oauth_required ?? false,
checked_at: health.checked_at,
error: health.error,
Expand Down
Loading
Loading