From b7e8254a6de612ef5561c95e02d2e2215dc76a51 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 18 Sep 2026 08:09:27 +0000 Subject: [PATCH] fix(auth): preserve queued refresh credential provenance --- server/src/capabilities.ts | 38 +- server/src/crawler.ts | 121 ++- server/src/db/agent-snapshot-db.ts | 20 +- server/src/db/compliance-db.ts | 22 +- .../src/db/compliance-refresh-requests-db.ts | 169 +++- server/src/db/federated-index-db.ts | 7 +- ...iance_refresh_authenticated_credential.sql | 100 +++ server/src/db/type-reclassification-log-db.ts | 14 +- server/src/federated-index.ts | 6 +- server/src/health.ts | 40 +- .../routes/helpers/resolve-user-agent-auth.ts | 10 + server/src/routes/registry-api.ts | 181 ++-- server/src/services/agent-ownership.ts | 13 +- server/src/services/badge-issuance.ts | 2 + .../compliance-refresh-authorization.ts | 212 +++++ .../src/services/compliance-refresh-queue.ts | 40 +- ...iance-refresh-credential-migration.test.ts | 307 +++++++ .../compliance-refresh-requests-queue.test.ts | 187 +++- .../registry-api-agent-refresh.test.ts | 799 +++++++++++------- server/tests/unit/badge-fan-out.test.ts | 20 + ...lity-discovery-auth-classification.test.ts | 87 +- .../compliance-db-authorization-guard.test.ts | 54 ++ .../compliance-refresh-authorization.test.ts | 184 ++++ .../unit/compliance-refresh-queue.test.ts | 107 ++- .../crawler-refresh-authorization.test.ts | 178 ++++ .../unit/resolve-user-agent-auth.test.ts | 26 + .../tests/unit/workos-client-timeout.test.ts | 11 + 27 files changed, 2444 insertions(+), 511 deletions(-) create mode 100644 server/src/db/migrations/599_compliance_refresh_authenticated_credential.sql create mode 100644 server/src/services/compliance-refresh-authorization.ts create mode 100644 server/tests/integration/compliance-refresh-credential-migration.test.ts create mode 100644 server/tests/unit/compliance-db-authorization-guard.test.ts create mode 100644 server/tests/unit/compliance-refresh-authorization.test.ts create mode 100644 server/tests/unit/crawler-refresh-authorization.test.ts diff --git a/server/src/capabilities.ts b/server/src/capabilities.ts index 8afab664cc4..8d0f66cec4a 100644 --- a/server/src/capabilities.ts +++ b/server/src/capabilities.ts @@ -5,6 +5,7 @@ import { AAO_UA_DISCOVERY } from "./config/user-agents.js"; import { logOutboundRequest } from "./db/outbound-log-db.js"; import { agentConfigAuthFields, type SdkAuth } from "./services/sdk-auth-adapter.js"; import { withSdkSafeTransport } from "./utils/sdk-safe-fetch.js"; +import { isComplianceRefreshAccessFailure } from "./services/compliance-refresh-authorization.js"; import Ajv, { type ValidateFunction } from "ajv"; import addFormats from "ajv-formats"; import { readFileSync } from "node:fs"; @@ -473,7 +474,7 @@ export class CapabilityDiscovery { constructor() {} - async discoverCapabilities(agent: Agent, auth?: SdkAuth, forceRefresh = false): Promise { + async discoverCapabilities(agent: Agent, auth?: SdkAuth, forceRefresh = false, checkpoint?: () => Promise): Promise { // Skip cache when auth is provided — manual owner-triggered refresh // wants fresh data and may previously have cached an unauthed // discovery_error result. Periodic crawls (no auth) keep the cache. @@ -491,7 +492,8 @@ export class CapabilityDiscovery { const startTime = Date.now(); try { const protocol = agent.protocol || "mcp"; - const tools = await this.discoverTools(agent.url, protocol, auth); + const tools = await this.discoverTools(agent.url, protocol, auth, checkpoint); + await checkpoint?.(); logOutboundRequest({ agent_url: agent.url, @@ -520,11 +522,11 @@ export class CapabilityDiscovery { // deadline can cause the probe to time out on a healthy but slow endpoint. const hasGetAdcpCaps = toolNames.has('get_adcp_capabilities'); const rawAdcpCaps: Record | undefined = hasGetAdcpCaps - ? await this.fetchRawAdcpCapabilities(agent, auth) + ? await this.fetchRawAdcpCapabilities(agent, auth, checkpoint) : undefined; if (CapabilityDiscovery.CREATIVE_TOOLS.some(t => toolNames.has(t))) { - const creativeResult = await this.analyzeCreativeCapabilities(agent, tools, auth, rawAdcpCaps); + const creativeResult = await this.analyzeCreativeCapabilities(agent, tools, auth, rawAdcpCaps, checkpoint); profile.creative_capabilities = creativeResult.capabilities; if (creativeResult.probeFailed) profile.creative_capabilities_probe_failed = true; } @@ -548,9 +550,12 @@ export class CapabilityDiscovery { // tool list an agent advertises behind auth may differ from the // public-facing one, and the cache is read by unauthed periodic // crawls + the public registry render. + await checkpoint?.(); if (!auth) this.cache.set(agent.url, profile); return profile; } catch (error: any) { + if (isComplianceRefreshAccessFailure(error)) throw error; + await checkpoint?.(); logOutboundRequest({ agent_url: agent.url, request_type: 'discovery', @@ -578,15 +583,15 @@ export class CapabilityDiscovery { } } - private async discoverTools(url: string, protocol: "mcp" | "a2a", auth?: SdkAuth): Promise { + private async discoverTools(url: string, protocol: "mcp" | "a2a", auth?: SdkAuth, checkpoint?: () => Promise): Promise { if (protocol === "a2a") { - return this.discoverA2ATools(url, auth); + return this.discoverA2ATools(url, auth, checkpoint); } else { - return this.discoverMCPTools(url, auth); + return this.discoverMCPTools(url, auth, checkpoint); } } - private async discoverMCPTools(url: string, auth?: SdkAuth): Promise { + private async discoverMCPTools(url: string, auth?: SdkAuth, checkpoint?: () => Promise): Promise { try { // Use AdCPClient to connect to agent const { AdCPClient } = await import("@adcp/sdk"); @@ -606,6 +611,7 @@ export class CapabilityDiscovery { })); const client = multiClient.agent("discovery"); + await checkpoint?.(); const agentInfo = await client.getAgentInfo(); logger.debug({ url, toolCount: agentInfo.tools.length }, 'MCP discovery completed'); @@ -616,6 +622,7 @@ export class CapabilityDiscovery { verified_at: new Date().toISOString(), })); } catch (error: any) { + if (isComplianceRefreshAccessFailure(error)) throw error; // Re-throw AuthenticationRequiredError to preserve OAuth metadata for callers if (error instanceof AuthenticationRequiredError) { logger.info({ url, hasOAuth: error.hasOAuth }, 'MCP agent requires OAuth authentication'); @@ -633,7 +640,7 @@ export class CapabilityDiscovery { } } - private async discoverA2ATools(url: string, auth?: SdkAuth): Promise { + private async discoverA2ATools(url: string, auth?: SdkAuth, checkpoint?: () => Promise): Promise { try { // Use AdCPClient to connect to agent const { AdCPClient } = await import("@adcp/sdk"); @@ -650,6 +657,7 @@ export class CapabilityDiscovery { })); const client = multiClient.agent("discovery"); + await checkpoint?.(); const agentInfo = await client.getAgentInfo(); logger.debug({ url, toolCount: agentInfo.tools.length }, 'A2A discovery completed'); @@ -660,6 +668,7 @@ export class CapabilityDiscovery { verified_at: new Date().toISOString(), })); } catch (error: any) { + if (isComplianceRefreshAccessFailure(error)) throw error; // Re-throw AuthenticationRequiredError to preserve OAuth metadata for callers if (error instanceof AuthenticationRequiredError) { logger.info({ url, hasOAuth: error.hasOAuth }, 'A2A agent requires OAuth authentication'); @@ -726,6 +735,7 @@ export class CapabilityDiscovery { tools: ToolCapability[], auth?: SdkAuth, rawAdcpCaps?: Record, + checkpoint?: () => Promise, ): Promise<{ capabilities: CreativeCapabilities; probeFailed: boolean }> { const toolNames = new Set(tools.map((t) => t.name.toLowerCase())); let declared: { ok: true; capabilities?: CreativeCapabilities } | { ok: false }; @@ -743,7 +753,7 @@ export class CapabilityDiscovery { } } } else { - declared = await this.fetchCreativeCapabilities(agent, auth); + declared = await this.fetchCreativeCapabilities(agent, auth, checkpoint); } } else { declared = { ok: true, capabilities: undefined }; @@ -774,6 +784,7 @@ export class CapabilityDiscovery { private async fetchCreativeCapabilities( agent: Agent, auth?: SdkAuth, + checkpoint?: () => Promise, ): Promise<{ ok: true; capabilities?: CreativeCapabilities } | { ok: false }> { try { const { AdCPClient } = await import("@adcp/sdk"); @@ -788,7 +799,9 @@ export class CapabilityDiscovery { transport: { maxResponseBytes: 1024 * 1024 }, })); const client = multiClient.agent("discovery"); + await checkpoint?.(); const result = await client.getAdcpCapabilities({}, undefined, { timeout: 10_000 }); + await checkpoint?.(); if (!result?.success) { logger.debug({ url: agent.url, error: result?.error }, 'Creative capability probe returned an error result'); return { ok: false }; @@ -797,6 +810,7 @@ export class CapabilityDiscovery { if (creative === undefined || creative === null) return { ok: true }; return { ok: true, capabilities: await sanitizeCreativeCapabilities(creative) }; } catch (err: any) { + if (isComplianceRefreshAccessFailure(err)) throw err; logger.debug({ url: agent.url, err: err?.message }, 'Creative capability fetch failed'); return { ok: false }; } @@ -816,6 +830,7 @@ export class CapabilityDiscovery { private async fetchRawAdcpCapabilities( agent: Agent, auth?: SdkAuth, + checkpoint?: () => Promise, ): Promise | undefined> { try { const { AdCPClient } = await import("@adcp/sdk"); @@ -830,10 +845,13 @@ export class CapabilityDiscovery { transport: { maxResponseBytes: 1024 * 1024 }, })); const client = multiClient.agent("discovery"); + await checkpoint?.(); const result = await client.getAdcpCapabilities({}, undefined, { timeout: 10_000 }); + await checkpoint?.(); if (!result?.success) return undefined; return result.data as Record | undefined; } catch (err: any) { + if (isComplianceRefreshAccessFailure(err)) throw err; logger.debug({ url: agent.url, err: err?.message }, 'get_adcp_capabilities fetch failed'); return undefined; } diff --git a/server/src/crawler.ts b/server/src/crawler.ts index 197e3fe1bb4..8f54282cc5f 100644 --- a/server/src/crawler.ts +++ b/server/src/crawler.ts @@ -1,5 +1,5 @@ import type { Agent, FederatedAgent, ResolvedBrand } from "./types.js"; -import type { Client } from "pg"; +import type { Client, PoolClient } from "pg"; import { PropertyCrawler, getPropertyIndex, type AgentInfo, type CrawlResult } from "@adcp/sdk"; import { sanitizeAdagentsProperty } from "./discovery/property-index-guard.js"; import { FederatedIndexService } from "./federated-index.js"; @@ -34,12 +34,13 @@ import { type AgentInventoryProfilesDatabase, type ProfileUpsertInput, } from "./db/agent-inventory-profiles-db.js"; -import { getDedicatedClient, query, withDatabaseDeadline } from "./db/client.js"; +import { getClient, getDedicatedClient, query, withDatabaseDeadline } from "./db/client.js"; import { PropertyDatabase } from "./db/property-db.js"; import { verifyHostedPropertyOrigin } from "./services/hosted-property-origin-verifier.js"; import { insertTypeReclassification } from "./db/type-reclassification-log-db.js"; import { resolveUserAgentAuth } from "./routes/helpers/resolve-user-agent-auth.js"; import { adaptAuthForSdk, type SdkAuth } from "./services/sdk-auth-adapter.js"; +import { isComplianceRefreshAccessFailure, type ComplianceRefreshAuthorizationGuard } from "./services/compliance-refresh-authorization.js"; import { PublisherCrawlRequestsDatabase, type CreatePublisherCrawlRequestInput, @@ -1747,7 +1748,11 @@ export class CrawlerService { * route handler maps that to a 502 so the user sees why the refresh * couldn't happen (timeout, DNS, OAuth wall, etc). */ - async refreshSingleAgent(agentUrl: string, options: { auth?: SdkAuth; ownerOrgId?: string } = {}): Promise<{ + async refreshSingleAgent(agentUrl: string, options: { + auth?: SdkAuth; + ownerOrgId?: string; + authorization?: ComplianceRefreshAuthorizationGuard; + } = {}): Promise<{ online: boolean; tools_count: number | null; response_time_ms: number | null; @@ -1758,7 +1763,7 @@ export class CrawlerService { error?: string; }> { const PROBE_TIMEOUT_MS = 10000; - const { auth, ownerOrgId } = options; + const { auth, ownerOrgId, authorization } = options; const pausedUrls = await this.getPausedAgentUrls(); if (pausedUrls.has(agentUrl)) { @@ -1792,12 +1797,17 @@ export class CrawlerService { // with no saved owner auth would still read the shared 15-minute // capability/health cache and keep reporting a fixed-then-redeployed // issue as unresolved (#5777). + await authorization?.checkpoint(); const profile = await Promise.race([ - this.capabilityDiscovery.discoverCapabilities(agent, auth, true), + this.capabilityDiscovery.discoverCapabilities(agent, auth, true, authorization?.checkpoint), new Promise((_, reject) => setTimeout(() => reject(new Error('Probe timeout')), PROBE_TIMEOUT_MS) ), ]); + await authorization?.checkpoint(); + if (authorization && profile.agent_url !== agentUrl) { + throw Object.assign(new Error('Capability profile did not match the refresh target'), { code: 'probe_failed' }); + } const inferredType = this.capabilityDiscovery.inferTypeFromProfile(profile); const effectiveType = knownType || inferredType; @@ -1805,31 +1815,29 @@ export class CrawlerService { const [health, stats] = await Promise.all([ Promise.race([ - this.healthChecker.checkHealth(agentForHealth, auth, true), + this.healthChecker.checkHealth(agentForHealth, auth, true, authorization?.checkpoint), new Promise((_, reject) => setTimeout(() => reject(new Error('Health timeout')), PROBE_TIMEOUT_MS) ), - ]).catch((err): import('./types.js').AgentHealth => ({ - online: false, - checked_at: new Date().toISOString(), - error: err instanceof Error ? err.message : 'health check failed', - })), + ]).catch((err): import('./types.js').AgentHealth => { + if (isComplianceRefreshAccessFailure(err)) throw err; + return { + online: false, + checked_at: new Date().toISOString(), + error: err instanceof Error ? err.message : 'health check failed', + }; + }), Promise.race([ - this.healthChecker.getStats(agentForHealth, auth, true), + this.healthChecker.getStats(agentForHealth, auth, true, authorization?.checkpoint), new Promise((_, reject) => setTimeout(() => reject(new Error('Stats timeout')), PROBE_TIMEOUT_MS) ), - ]).catch((): import('./types.js').AgentStats => ({})), - ]); - - await Promise.all([ - this.snapshotDb.upsertCapabilities( - profile, - inferredType === 'unknown' ? null : inferredType, - { trackUnknownProbe: !knownType }, - ), - this.snapshotDb.upsertHealth(agentUrl, health, stats), + ]).catch((err): import('./types.js').AgentStats => { + if (isComplianceRefreshAccessFailure(err)) throw err; + return {}; + }), ]); + await authorization?.checkpoint(); // Same type-promotion policy as refreshAgentSnapshots: promote when // stored is unknown; log disagreement without auto-flipping (see #3538). @@ -1837,27 +1845,52 @@ export class CrawlerService { const isDisagreement = !!knownType && inferredType !== 'unknown' && knownType !== inferredType; - if (isDisagreement) { - log.warn( - { url: agentUrl, knownType, inferredType }, - 'Agent type disagreement: stored vs probed. Run backfill to reconcile.' - ); - await insertTypeReclassification({ - agentUrl, - oldType: knownType ?? null, - newType: inferredType, - source: 'crawler_promote', - notes: { decision: 'logged_only_no_promote', triggered_by: 'manual_refresh' }, - }); - } - - let typePromoted = false; - if (canPromote) { - await this.federatedIndex.updateAgentMetadata(agentUrl, { - agent_type: inferredType, - protocol: profile.protocol, - }); - typePromoted = true; + const writeSnapshotsAndType = async (client?: PoolClient) => { + await Promise.all([ + this.snapshotDb.upsertCapabilities( + profile, + inferredType === 'unknown' ? null : inferredType, + { trackUnknownProbe: !knownType, ...(client ? { client } : {}) }, + ), + this.snapshotDb.upsertHealth(agentUrl, health, stats, client), + ]); + if (isDisagreement) { + log.warn( + { url: agentUrl, knownType, inferredType }, + 'Agent type disagreement: stored vs probed. Run backfill to reconcile.' + ); + await insertTypeReclassification({ + agentUrl, + oldType: knownType ?? null, + newType: inferredType, + source: 'crawler_promote', + notes: { decision: 'logged_only_no_promote', triggered_by: 'manual_refresh' }, + }, client); + } + if (canPromote) { + await this.federatedIndex.updateAgentMetadata(agentUrl, { + agent_type: inferredType, + protocol: profile.protocol, + }, client); + } + }; + if (authorization) { + // Keep authorization locks and every mutation on the same connection. + // Losing this transaction cannot leave a detached writer committing. + const client = await getClient(); + try { + await client.query('BEGIN'); + await authorization.beforeWrite(client, agentUrl); + await writeSnapshotsAndType(client); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined); + throw error; + } finally { + client.release(); + } + } else { + await writeSnapshotsAndType(); } return { @@ -1865,7 +1898,7 @@ export class CrawlerService { tools_count: health.tools_count ?? null, response_time_ms: health.response_time_ms ?? null, inferred_type: inferredType, - type_promoted: typePromoted, + type_promoted: canPromote, oauth_required: profile.oauth_required ?? false, checked_at: health.checked_at, error: health.error, diff --git a/server/src/db/agent-snapshot-db.ts b/server/src/db/agent-snapshot-db.ts index 95c9b121d56..2b24294cffa 100644 --- a/server/src/db/agent-snapshot-db.ts +++ b/server/src/db/agent-snapshot-db.ts @@ -1,4 +1,5 @@ import { query } from './client.js'; +import type { PoolClient } from 'pg'; import { createLogger } from '../logger.js'; import type { AgentHealth, AgentStats } from '../types.js'; import type { @@ -106,9 +107,11 @@ export class AgentSnapshotDatabase { return map; } - async upsertHealth(agentUrl: string, health: AgentHealth, stats: AgentStats): Promise { + /** A supplied client belongs to a guarded transaction and must observe failures. */ + async upsertHealth(agentUrl: string, health: AgentHealth, stats: AgentStats, client?: PoolClient): Promise { + const executeQuery: typeof query = client ? (text, params) => client.query(text, params) : query; try { - await query( + await executeQuery( `INSERT INTO agent_health_snapshot (agent_url, online, response_time_ms, tools_count, resources_count, error, checked_at, stats_json, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) @@ -133,6 +136,7 @@ export class AgentSnapshotDatabase { ], ); } catch (err) { + if (client) throw err; logger.warn({ agentUrl, err }, 'Failed to upsert agent health snapshot'); } } @@ -140,18 +144,21 @@ export class AgentSnapshotDatabase { async upsertCapabilities( profile: AgentCapabilityProfile, inferredType: string | null, - options: { trackUnknownProbe?: boolean } = {}, + options: { trackUnknownProbe?: boolean; client?: PoolClient } = {}, ): Promise { + const client = options.client; + const executeQuery: typeof query = client ? (text, params) => client.query(text, params) : query; try { const trackUnknownProbe = options.trackUnknownProbe ?? true; const unknownState = inferredType === null && trackUnknownProbe ? await this.nextUnknownProbeState( profile.agent_url, profile.discovery_error ? 'unreachable' : 'unclassifiable', + options.client, ) : null; - await query( + await executeQuery( `INSERT INTO agent_capabilities_snapshot (agent_url, protocol, discovered_tools_json, standard_operations_json, creative_capabilities_json, signals_capabilities_json, @@ -223,6 +230,7 @@ export class AgentSnapshotDatabase { ], ); } catch (err) { + if (options.client) throw err; logger.warn({ agentUrl: profile.agent_url, err }, 'Failed to upsert agent capabilities snapshot'); } } @@ -268,8 +276,10 @@ export class AgentSnapshotDatabase { private async nextUnknownProbeState( agentUrl: string, terminalStateOnExhaustion: UnknownProbeTerminalState, + client?: PoolClient, ): Promise { - const result = await query<{ unknown_probe_attempt_count: number }>( + const executeQuery: typeof query = client ? (text, params) => client.query(text, params) : query; + const result = await executeQuery<{ unknown_probe_attempt_count: number }>( `SELECT unknown_probe_attempt_count FROM agent_capabilities_snapshot WHERE agent_url = $1`, diff --git a/server/src/db/compliance-db.ts b/server/src/db/compliance-db.ts index c87302da539..4ec261d7fc5 100644 --- a/server/src/db/compliance-db.ts +++ b/server/src/db/compliance-db.ts @@ -6,6 +6,7 @@ import { logger as baseLogger } from '../logger.js'; import { CatalogEventsDatabase } from './catalog-events-db.js'; import { ComplianceRefreshLeaseLostError } from './compliance-refresh-requests-db.js'; import type { VerificationProfileRoleAssessmentInput } from '../services/verification-profile-assessment.js'; +import { isComplianceRefreshAccessFailure, type ComplianceRefreshWriteGuard } from '../services/compliance-refresh-authorization.js'; const logger = baseLogger.child({ module: 'compliance-db' }); const catalogEventsDb = new CatalogEventsDatabase(); @@ -352,6 +353,7 @@ interface BadgeGradingGuard { // ===================================================== export class ComplianceDatabase { + constructor(private readonly beforeCanonicalWrite?: ComplianceRefreshWriteGuard) {} // ----- Registry Metadata ----- @@ -506,6 +508,7 @@ export class ComplianceDatabase { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, agentUrl); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${agentUrl}`], @@ -561,6 +564,7 @@ export class ComplianceDatabase { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, agentUrl); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${agentUrl}`], @@ -622,6 +626,7 @@ export class ComplianceDatabase { try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, input.agent_url); // Profile selection and authoritative evidence publication share this // lock. A selection can therefore only commit against the latest fully @@ -1745,8 +1750,9 @@ export class ComplianceDatabase { * access token as a bearer so callers surface a clear 401 from the agent * rather than sending no Authorization header at all. */ - async resolveOwnerAuth(agentUrl: string): Promise { + async resolveOwnerAuth(agentUrl: string, checkpoint?: () => Promise): Promise { try { + await checkpoint?.(); const result = await query( `SELECT ac.organization_id, ac.auth_token_encrypted, ac.auth_token_iv, ac.auth_type, @@ -1777,6 +1783,7 @@ export class ComplianceDatabase { [agentUrl, JSON.stringify([{ url: agentUrl }])], ); + await checkpoint?.(); const row = result.rows[0]; if (!row) return undefined; @@ -1890,6 +1897,7 @@ export class ComplianceDatabase { return undefined; } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; logger.warn({ err: error, agentUrl }, 'Could not resolve owner auth'); return undefined; } @@ -1921,6 +1929,7 @@ export class ComplianceDatabase { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, badge.agent_url); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${badge.agent_url}`], @@ -2123,10 +2132,11 @@ export class ComplianceDatabase { expectedGeneration?: string, gradingGuard?: BadgeGradingGuard, ): Promise { - if (expectedGeneration !== undefined) { + if (expectedGeneration !== undefined || this.beforeCanonicalWrite) { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, agentUrl); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${agentUrl}`], @@ -2142,7 +2152,7 @@ export class ComplianceDatabase { verification_token = NULL, token_expires_at = NULL, updated_at = NOW() WHERE agent_url = $1 AND role = $2 AND adcp_version = $3 AND status IN ('active', 'degraded') - AND COALESCE(( + AND ($5::bigint IS NULL OR COALESCE(( SELECT badge_requalification_generation FROM agent_registry_metadata WHERE agent_url = $1 ), 0) = $5::bigint @@ -2237,6 +2247,7 @@ export class ComplianceDatabase { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, agentUrl); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${agentUrl}`], @@ -2386,10 +2397,11 @@ export class ComplianceDatabase { expectedGeneration?: string, gradingGuard?: BadgeGradingGuard, ): Promise { - if (expectedGeneration !== undefined) { + if (expectedGeneration !== undefined || this.beforeCanonicalWrite) { const client = await getClient(); try { await client.query('BEGIN'); + await this.beforeCanonicalWrite?.(client, agentUrl); await client.query( 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [`verification-badge:${agentUrl}`], @@ -2405,7 +2417,7 @@ export class ComplianceDatabase { verification_token = NULL, token_expires_at = NULL, updated_at = NOW() WHERE agent_url = $1 AND role = $2 AND adcp_version = $3 AND status = 'active' - AND COALESCE(( + AND ($4::bigint IS NULL OR COALESCE(( SELECT badge_requalification_generation FROM agent_registry_metadata WHERE agent_url = $1 ), 0) = $4::bigint diff --git a/server/src/db/compliance-refresh-requests-db.ts b/server/src/db/compliance-refresh-requests-db.ts index 3c356a726e3..27e839b7f75 100644 --- a/server/src/db/compliance-refresh-requests-db.ts +++ b/server/src/db/compliance-refresh-requests-db.ts @@ -1,3 +1,4 @@ +import type { PoolClient } from 'pg'; import { getClient, getDedicatedClient, query, withDatabaseDeadline } from './client.js'; const DB_DEADLINE_MS = 5_000; @@ -12,6 +13,8 @@ export interface ComplianceRefreshRequest { owner_org_id: string | null; requester_type: 'user' | 'static_admin'; requested_by_user_id: string | null; + requested_by_auth_workos_user_id: string; + authorization_fingerprint: string; triggered_by: 'owner_test' | 'manual'; test_session_id: string; status: ComplianceRefreshRequestStatus; @@ -48,6 +51,8 @@ export interface CreateComplianceRefreshRequestInput { ownerOrgId: string | null; requesterType: 'user' | 'static_admin'; requestedByUserId: string | null; + requestedByAuthWorkosUserId: string | null; + authorizationFingerprint: string; triggeredBy: 'owner_test' | 'manual'; agentWindowMs?: number; requesterWindowMs?: number; @@ -82,6 +87,71 @@ export class ComplianceRefreshInProgressError extends Error { } } +export class ComplianceRefreshProvenanceError extends Error { + readonly code = 'authorization_provenance_missing'; + + constructor() { + super('Authenticated credential provenance is missing; submit a new refresh'); + this.name = 'ComplianceRefreshProvenanceError'; + } +} + +/** Legacy requester IDs may be canonical identities; they are never provenance. */ +export function hasComplianceRefreshProvenance(request: Pick): boolean { + const credential = request.requested_by_auth_workos_user_id; + return request.requester_type === 'user' + && typeof credential === 'string' + && credential.length > 0 + && credential !== '__unproven__' + && credential === credential.trim() + && request.requested_by_user_id === credential + && isComplianceRefreshAuthorizationFingerprint(credential, request.authorization_fingerprint) + && ((request.triggered_by === 'manual' && request.owner_org_id === null) + || (request.triggered_by === 'owner_test' + && typeof request.owner_org_id === 'string' && request.owner_org_id.trim().length > 0)); +} + +export function isComplianceRefreshAuthorizationFingerprint(credentialId: string, value: unknown): value is string { + if (value === '') return true; + if (typeof value !== 'string' || !value.startsWith(`${credentialId}:`)) return false; + const epoch = value.slice(credentialId.length + 1); + return /^[1-9][0-9]{0,18}$/.test(epoch) && BigInt(epoch) <= 9223372036854775807n; +} + +function authorizationStateError(code: 'authorization_revoked' | 'authorization_unavailable'): Error & { code: typeof code } { + return Object.assign(new Error(code === 'authorization_revoked' + ? 'Authorization changed before the refresh started' + : 'Refresh authorization is temporarily unavailable'), { code }); +} + +/** Fence both existing epoch changes and the first epoch INSERT until commit. */ +export async function assertComplianceRefreshAuthorizationFingerprint( + client: PoolClient, + credentialId: string, + expected: string, +): Promise { + let current: string; + try { + const user = await client.query( + 'SELECT workos_user_id FROM users WHERE workos_user_id = $1 FOR UPDATE', [credentialId], + ); + if (!user.rows.length) throw authorizationStateError('authorization_unavailable'); + const epoch = await client.query<{ epoch: string }>( + 'SELECT epoch::text AS epoch FROM authorization_epochs WHERE workos_user_id = $1 FOR SHARE', [credentialId], + ); + current = epoch.rows.length ? `${credentialId}:${epoch.rows[0].epoch}` : ''; + if (!isComplianceRefreshAuthorizationFingerprint(credentialId, current)) { + throw authorizationStateError('authorization_unavailable'); + } + } catch { + throw authorizationStateError('authorization_unavailable'); + } + if (current !== expected) throw authorizationStateError('authorization_revoked'); +} + export class ComplianceRefreshLeaseLostError extends Error { readonly code = 'lease_lost'; @@ -106,6 +176,24 @@ export class ComplianceRefreshRequestsDatabase { request: ComplianceRefreshRequest; coalesced: boolean; }> { + // Validate before coalescing: an existing row must never make malformed + // caller provenance acceptable, even when no INSERT would be performed. + if (!hasComplianceRefreshProvenance({ + requester_type: input.requesterType, + requested_by_user_id: input.requestedByUserId, + requested_by_auth_workos_user_id: input.requestedByAuthWorkosUserId!, + authorization_fingerprint: input.authorizationFingerprint, + triggered_by: input.triggeredBy, + owner_org_id: input.ownerOrgId, + })) throw new ComplianceRefreshProvenanceError(); + + const sameCredentialContext = (request: ComplianceRefreshRequest): boolean => + hasComplianceRefreshProvenance(request) + && request.triggered_by === input.triggeredBy + && request.owner_org_id === input.ownerOrgId + && request.requester_type === input.requesterType + && request.requested_by_auth_workos_user_id === input.requestedByAuthWorkosUserId + && request.authorization_fingerprint === input.authorizationFingerprint; const agentWindowMs = input.agentWindowMs ?? 60_000; const requesterWindowMs = input.requesterWindowMs ?? 60 * 60_000; const requesterLimit = input.requesterLimit ?? 30; @@ -118,6 +206,9 @@ export class ComplianceRefreshRequestsDatabase { transactionStarted = true; await client.query("SELECT set_config('statement_timeout', '5000ms', true)"); await client.query("SELECT set_config('lock_timeout', '2000ms', true)"); + await assertComplianceRefreshAuthorizationFingerprint( + client, input.requestedByAuthWorkosUserId!, input.authorizationFingerprint, + ); await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [ `agent-compliance-refresh:${input.agentUrl}`, ]); @@ -127,17 +218,35 @@ export class ComplianceRefreshRequestsDatabase { FROM agent_compliance_refresh_requests WHERE agent_url = $1 AND status IN ('queued', 'running') ORDER BY created_at ASC - LIMIT 1`, + LIMIT 1 FOR UPDATE`, [input.agentUrl], ); if (active.rows[0]) { - const sameCredentialContext = active.rows[0].triggered_by === input.triggeredBy - && active.rows[0].owner_org_id === input.ownerOrgId; - if (!sameCredentialContext) { + const provenanceMissing = !hasComplianceRefreshProvenance(active.rows[0]); + const epochRevoked = active.rows[0].requested_by_auth_workos_user_id === input.requestedByAuthWorkosUserId + && active.rows[0].authorization_fingerprint !== input.authorizationFingerprint; + if (provenanceMissing || epochRevoked) { + // Do not replace an operation while a worker still owns its fence. + const rejected = await client.query( + `UPDATE agent_compliance_refresh_requests + SET status = 'failed', completed_at = NOW(), updated_at = NOW(), + lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL, heartbeat_at = NULL, + last_error_code = $2, last_error = $3 + WHERE id = $1 AND pg_try_advisory_xact_lock( + hashtextextended('compliance-refresh-fence:' || id::text, 0) + )`, + [active.rows[0].id, + provenanceMissing ? 'authorization_provenance_missing' : 'authorization_revoked', + provenanceMissing ? 'Authenticated credential provenance is missing; submit a new refresh' + : 'Authorization changed before the refresh started'], + ); + if (!rejected.rowCount) throw new ComplianceRefreshInProgressError(); + } else if (!sameCredentialContext(active.rows[0])) { throw new ComplianceRefreshInProgressError(); + } else { + await client.query('COMMIT'); + return { request: active.rows[0], coalesced: true }; } - await client.query('COMMIT'); - return { request: active.rows[0], coalesced: true }; } await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', [ @@ -156,15 +265,18 @@ export class ComplianceRefreshRequestsDatabase { `SELECT *, created_at + ($2::double precision * INTERVAL '1 millisecond') AS retry_at FROM agent_compliance_refresh_requests WHERE agent_url = $1 + AND status IN ('succeeded', 'failed') + AND requested_by_auth_workos_user_id <> '__unproven__' + AND last_error_code IS DISTINCT FROM 'authorization_provenance_missing' + AND last_error_code IS DISTINCT FROM 'authorization_revoked' + AND NOT (requested_by_auth_workos_user_id = $3 AND authorization_fingerprint <> $4) AND created_at > NOW() - ($2::double precision * INTERVAL '1 millisecond') ORDER BY created_at DESC LIMIT 1`, - [input.agentUrl, agentWindowMs], + [input.agentUrl, agentWindowMs, input.requestedByAuthWorkosUserId, input.authorizationFingerprint], ); - if (recent.rows[0]) { - const sameCredentialContext = recent.rows[0].triggered_by === input.triggeredBy - && recent.rows[0].owner_org_id === input.ownerOrgId; - if (sameCredentialContext) { + if (recent.rows[0] && hasComplianceRefreshProvenance(recent.rows[0])) { + if (sameCredentialContext(recent.rows[0])) { await client.query('COMMIT'); return { request: recent.rows[0], coalesced: true }; } @@ -188,8 +300,12 @@ export class ComplianceRefreshRequestsDatabase { FROM agent_compliance_refresh_requests WHERE requester_type = $1 AND requested_by_user_id IS NOT DISTINCT FROM $2 + AND requested_by_auth_workos_user_id = $2 + AND authorization_fingerprint = $4 + AND last_error_code IS DISTINCT FROM 'authorization_provenance_missing' + AND last_error_code IS DISTINCT FROM 'authorization_revoked' AND created_at > NOW() - ($3::double precision * INTERVAL '1 millisecond')`, - [input.requesterType, input.requestedByUserId, requesterWindowMs], + [input.requesterType, input.requestedByUserId, requesterWindowMs, input.authorizationFingerprint], ); const requester = requesterState.rows[0]; if (Number(requester?.active_count ?? 0) >= requesterActiveLimit) { @@ -206,8 +322,9 @@ export class ComplianceRefreshRequestsDatabase { const inserted = await client.query( `INSERT INTO agent_compliance_refresh_requests - (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, test_session_id) - VALUES ($1, $2, $3, $4, $5, $6, $7) + (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, test_session_id, + requested_by_auth_workos_user_id, authorization_fingerprint) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *`, [ input.id, @@ -217,6 +334,8 @@ export class ComplianceRefreshRequestsDatabase { input.requestedByUserId, input.triggeredBy, `owner-refresh-${input.id}`, + input.requestedByAuthWorkosUserId, + input.authorizationFingerprint, ], ); await client.query('COMMIT'); @@ -250,6 +369,7 @@ export class ComplianceRefreshRequestsDatabase { transactionStarted = true; await client.query("SELECT set_config('statement_timeout', '5000ms', true)"); await client.query("SELECT set_config('lock_timeout', '2000ms', true)"); + await client.query("SELECT set_config('adcp.compliance_refresh_writer_contract', 'authenticated-credential-v1', true)"); const expiredFinal = await client.query( `UPDATE agent_compliance_refresh_requests @@ -506,9 +626,18 @@ export class ComplianceRefreshRequestsDatabase { id: string, leaseToken: string, resultJson: Record, + beforeWrite: (client: PoolClient) => Promise, ): Promise { - return withDatabaseDeadline(Date.now() + DB_DEADLINE_MS, async () => { - const result = await query( + if (typeof beforeWrite !== 'function') throw authorizationStateError('authorization_unavailable'); + const client = await getClient(); + let transactionStarted = false; + try { + await client.query('BEGIN'); + transactionStarted = true; + await client.query("SELECT set_config('statement_timeout', '5000ms', true)"); + await client.query("SELECT set_config('lock_timeout', '2000ms', true)"); + await beforeWrite(client); + const result = await client.query( `UPDATE agent_compliance_refresh_requests SET status = 'succeeded', result_json = $3::jsonb, @@ -524,8 +653,14 @@ export class ComplianceRefreshRequestsDatabase { AND lease_expires_at > NOW()`, [id, leaseToken, JSON.stringify(resultJson)], ); + await client.query('COMMIT'); return (result.rowCount ?? 0) === 1; - }, { readOnly: false }); + } catch (error) { + if (transactionStarted) await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } } async markFailed( diff --git a/server/src/db/federated-index-db.ts b/server/src/db/federated-index-db.ts index b2f2e737bf7..75f859f8cc7 100644 --- a/server/src/db/federated-index-db.ts +++ b/server/src/db/federated-index-db.ts @@ -1,4 +1,5 @@ import { query, queryWithTimeout } from './client.js'; +import type { PoolClient } from 'pg'; import { canonicalizePublisherDomain } from '../services/publisher-domain.js'; /** @@ -773,9 +774,11 @@ export class FederatedIndexDatabase { */ async updateAgentMetadata( agentUrl: string, - metadata: { name?: string; agent_type?: string; protocol?: string } + metadata: { name?: string; agent_type?: string; protocol?: string }, + client?: PoolClient, ): Promise { - await query( + const executeQuery: typeof query = client ? (text, params) => client.query(text, params) : query; + await executeQuery( `UPDATE discovered_agents SET name = COALESCE($2, name), agent_type = COALESCE($3, agent_type), diff --git a/server/src/db/migrations/599_compliance_refresh_authenticated_credential.sql b/server/src/db/migrations/599_compliance_refresh_authenticated_credential.sql new file mode 100644 index 00000000000..de8d612c56b --- /dev/null +++ b/server/src/db/migrations/599_compliance_refresh_authenticated_credential.sql @@ -0,0 +1,100 @@ +-- Drain old application workers before applying this migration. An old +-- in-memory claim must not survive the authenticated-credential boundary. +LOCK TABLE agent_compliance_refresh_requests IN ACCESS EXCLUSIVE MODE; + +DO $$ +BEGIN + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'Compliance refresh credential migration requires READ COMMITTED'; + END IF; + IF EXISTS (SELECT 1 FROM agent_compliance_refresh_requests WHERE status = 'running') THEN + RAISE EXCEPTION 'Drain all running compliance refresh requests before applying migration 599'; + END IF; +END $$; + +-- Existing requester IDs can be canonical linked IDs. Sentinels record that +-- provenance is unknown; no credential or authorization epoch is backfilled. +ALTER TABLE agent_compliance_refresh_requests + ADD COLUMN requested_by_auth_workos_user_id TEXT NOT NULL DEFAULT '__unproven__', + ADD COLUMN authorization_fingerprint TEXT NOT NULL DEFAULT '__unproven__'; + +ALTER TABLE agent_compliance_refresh_requests + ALTER COLUMN requested_by_auth_workos_user_id DROP DEFAULT, + ALTER COLUMN authorization_fingerprint DROP DEFAULT; + +UPDATE agent_compliance_refresh_requests + SET status = 'failed', completed_at = NOW(), updated_at = NOW(), + lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL, heartbeat_at = NULL, + result_json = NULL, + last_error_code = 'authorization_provenance_missing', + last_error = 'Authenticated credential provenance is missing; submit a new refresh' + WHERE status = 'queued'; + +ALTER TABLE agent_compliance_refresh_requests + ADD CONSTRAINT agent_compliance_refresh_authenticated_credential CHECK ( + ( + requested_by_auth_workos_user_id = '__unproven__' + AND authorization_fingerprint = '__unproven__' + AND status IN ('succeeded', 'failed') + ) OR ( + requester_type = 'user' + AND requested_by_auth_workos_user_id <> '__unproven__' + AND requested_by_auth_workos_user_id = requested_by_user_id + AND char_length(requested_by_auth_workos_user_id) > 0 + AND requested_by_auth_workos_user_id = btrim(requested_by_auth_workos_user_id) + AND ( + authorization_fingerprint = '' + OR ( + left(authorization_fingerprint, char_length(requested_by_auth_workos_user_id) + 1) + = requested_by_auth_workos_user_id || ':' + AND substring(authorization_fingerprint FROM char_length(requested_by_auth_workos_user_id) + 2) + ~ '^[1-9][0-9]{0,18}$' + AND ( + char_length(substring(authorization_fingerprint FROM char_length(requested_by_auth_workos_user_id) + 2)) < 19 + OR substring(authorization_fingerprint FROM char_length(requested_by_auth_workos_user_id) + 2) COLLATE "C" + <= '9223372036854775807' COLLATE "C" + ) + ) + ) + ) + ); + +CREATE FUNCTION guard_compliance_refresh_authenticated_credential() +RETURNS TRIGGER LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.requester_type <> 'user' OR NEW.status <> 'queued' + OR NEW.requested_by_auth_workos_user_id = '__unproven__' + OR NEW.authorization_fingerprint = '__unproven__' THEN + RAISE EXCEPTION 'New compliance refresh requests require authenticated user provenance'; + END IF; + ELSE + IF ROW(NEW.id, NEW.agent_url, NEW.owner_org_id, NEW.requester_type, + NEW.requested_by_user_id, NEW.requested_by_auth_workos_user_id, + NEW.authorization_fingerprint, NEW.triggered_by, NEW.test_session_id) + IS DISTINCT FROM + ROW(OLD.id, OLD.agent_url, OLD.owner_org_id, OLD.requester_type, + OLD.requested_by_user_id, OLD.requested_by_auth_workos_user_id, + OLD.authorization_fingerprint, OLD.triggered_by, OLD.test_session_id) THEN + RAISE EXCEPTION 'Compliance refresh authenticated provenance is immutable'; + END IF; + -- Old binaries preserve new columns in UPDATEs, so NOT NULL alone does + -- not prevent them from claiming a request admitted by a new instance. + IF NEW.status = 'running' + AND (OLD.status <> 'running' OR NEW.lease_token IS DISTINCT FROM OLD.lease_token) + AND current_setting('adcp.compliance_refresh_writer_contract', true) + IS DISTINCT FROM 'authenticated-credential-v1' THEN + RAISE EXCEPTION 'Compliance refresh claims require the authenticated-credential worker contract'; + END IF; + END IF; + RETURN NEW; +END $$; + +CREATE TRIGGER guard_compliance_refresh_authenticated_credential + BEFORE INSERT OR UPDATE ON agent_compliance_refresh_requests + FOR EACH ROW EXECUTE FUNCTION guard_compliance_refresh_authenticated_credential(); + +COMMENT ON COLUMN agent_compliance_refresh_requests.requested_by_auth_workos_user_id IS + 'Exact authenticated WorkOS credential. __unproven__ marks historical requests that cannot execute or coalesce.'; +COMMENT ON COLUMN agent_compliance_refresh_requests.authorization_fingerprint IS + 'Immutable exact-credential authorization epoch at admission: empty if unbumped, credential:epoch otherwise. __unproven__ is historical only.'; diff --git a/server/src/db/type-reclassification-log-db.ts b/server/src/db/type-reclassification-log-db.ts index ed14affe4d9..5ad45ce1c1b 100644 --- a/server/src/db/type-reclassification-log-db.ts +++ b/server/src/db/type-reclassification-log-db.ts @@ -6,11 +6,11 @@ * insert helper. Idempotency at the row level means the caller decides what * counts as a "change" — we don't dedupe. * - * On insert failure we log and swallow — the audit log is observability, not - * a write barrier. A failed audit write must NOT roll back the caller's - * primary intent (a member-profile save, a crawl probe, a backfill row). + * Ordinary callers log and swallow insert failures. A caller-supplied + * transaction propagates failure so its guarded write can roll back. */ import { query } from './client.js'; +import type { PoolClient } from 'pg'; import { createLogger } from '../logger.js'; import { captureEvent } from '../utils/posthog.js'; @@ -32,10 +32,12 @@ export interface TypeReclassificationInsert { } export async function insertTypeReclassification( - entry: TypeReclassificationInsert + entry: TypeReclassificationInsert, + client?: PoolClient, ): Promise { + const executeQuery: typeof query = client ? (text, params) => client.query(text, params) : query; try { - await query( + await executeQuery( `INSERT INTO type_reclassification_log (agent_url, member_id, old_type, new_type, source, run_id, notes) VALUES ($1, $2, $3, $4, $5, $6, $7)`, @@ -50,6 +52,8 @@ export async function insertTypeReclassification( ] ); } catch (err) { + // A guarded caller owns the transaction; swallowing would hide rollback. + if (client) throw err; // PostgreSQL SQLSTATE codes are exactly 5 chars; the first 2 are the error // class (e.g. '23' = integrity_constraint_violation, '08' = connection_ // exception). The class is what ops actually alerts on — the full code is diff --git a/server/src/federated-index.ts b/server/src/federated-index.ts index 4f1a8f24680..79865dde368 100644 --- a/server/src/federated-index.ts +++ b/server/src/federated-index.ts @@ -2,6 +2,7 @@ import { FederatedIndexDatabase, type AgentPublisherAuthorization, type AgentPub import { MemberDatabase } from './db/member-db.js'; import { canonicalizeAgentUrl } from './db/publisher-db.js'; import type { FederatedAgent, FederatedPublisher, DomainLookupResult, AgentType } from './types.js'; +import type { PoolClient } from 'pg'; /** * Service layer for the federated agent/publisher registry. @@ -360,9 +361,10 @@ export class FederatedIndexService { */ async updateAgentMetadata( agentUrl: string, - metadata: { name?: string; agent_type?: string; protocol?: string } + metadata: { name?: string; agent_type?: string; protocol?: string }, + client?: PoolClient, ): Promise { - await this.db.updateAgentMetadata(agentUrl, metadata); + await this.db.updateAgentMetadata(agentUrl, metadata, client); } /** diff --git a/server/src/health.ts b/server/src/health.ts index acfaab695ba..7055f36c81d 100644 --- a/server/src/health.ts +++ b/server/src/health.ts @@ -9,6 +9,7 @@ import { logger } from "./logger.js"; import { promises as dnsPromises } from "node:dns"; import { agentConfigAuthFields, type SdkAuth } from "./services/sdk-auth-adapter.js"; import { withSdkSafeTransport } from "./utils/sdk-safe-fetch.js"; +import { isComplianceRefreshAccessFailure } from "./services/compliance-refresh-authorization.js"; export interface ClassifiedProbeError { kind: ProbeErrorKind; @@ -182,7 +183,7 @@ export class HealthChecker { this.formatsService = new FormatsService(); } - async checkHealth(agent: Agent, auth?: SdkAuth, forceRefresh = false): Promise { + async checkHealth(agent: Agent, auth?: SdkAuth, forceRefresh = false, checkpoint?: () => Promise): Promise { // Skip cache when auth is provided — manual owner-triggered refresh // wants fresh data. Periodic crawls (no auth) keep the cache. // `forceRefresh` covers the same intent for a manual refresh of an @@ -193,7 +194,8 @@ export class HealthChecker { if (cached) return cached; } - const health = await this.performHealthCheck(agent, auth); + const health = await this.performHealthCheck(agent, auth, checkpoint); + await checkpoint?.(); // Don't write authed probe results back to the shared cache — // tools_count discovered with credentials may differ from what // the agent exposes publicly, and the cache feeds unauthed reads. @@ -201,14 +203,14 @@ export class HealthChecker { return health; } - private async performHealthCheck(agent: Agent, auth?: SdkAuth): Promise { + private async performHealthCheck(agent: Agent, auth?: SdkAuth, checkpoint?: () => Promise): Promise { const startTime = Date.now(); const protocol = agent.protocol || "mcp"; // Only try the protocol the agent declares const health = protocol === "a2a" - ? await this.tryA2A(agent, startTime, auth) - : await this.tryMCP(agent, startTime, auth); + ? await this.tryA2A(agent, startTime, auth, checkpoint) + : await this.tryMCP(agent, startTime, auth, checkpoint); logOutboundRequest({ agent_url: agent.url, @@ -222,7 +224,7 @@ export class HealthChecker { return health; } - private async tryMCP(agent: Agent, startTime: number, auth?: SdkAuth): Promise { + private async tryMCP(agent: Agent, startTime: number, auth?: SdkAuth, checkpoint?: () => Promise): Promise { try { // Use AdCPClient to handle MCP protocol complexity (sessions, SSE, etc.) const { AdCPClient } = await import("@adcp/sdk"); @@ -235,6 +237,7 @@ export class HealthChecker { }], withSdkSafeTransport({ userAgent: AAO_UA_HEALTH_CHECK })); const client = multiClient.agent("health-check"); + await checkpoint?.(); const agentInfo = await client.getAgentInfo(); const responseTime = Date.now() - startTime; @@ -246,6 +249,8 @@ export class HealthChecker { resources_count: (agentInfo as any).resources?.length || 0, }; } catch (error: any) { + if (isComplianceRefreshAccessFailure(error)) throw error; + await checkpoint?.(); let classified = classifyMCPError(error); // Disambiguate `wrong_path` vs unreachable: when the SDK swallows // the DNS cause, a non-resolving host looks identical to a host @@ -258,7 +263,7 @@ export class HealthChecker { raw: classified.raw, }; } - const fallback = await this.tryHealthCheckFallback(agent, startTime, classified, auth); + const fallback = await this.tryHealthCheckFallback(agent, startTime, classified, auth, checkpoint); if (fallback) return fallback; return { online: false, @@ -285,9 +290,11 @@ export class HealthChecker { startTime: number, classified: ClassifiedProbeError, auth?: SdkAuth, + checkpoint?: () => Promise, ): Promise { if (!agent.health_check_url) return null; try { + await checkpoint?.(); const response = await safeFetch(agent.health_check_url, { method: "GET", headers: healthProbeHeaders(shouldSendHealthFallbackAuth(agent) ? auth : undefined), @@ -307,6 +314,7 @@ export class HealthChecker { error_detail: classified.raw, }; } catch (err) { + if (isComplianceRefreshAccessFailure(err)) throw err; // Don't surface fallback fetch failures to the dashboard — the MCP // error is the load-bearing signal. Log for debuggability. logger.debug({ err, agentUrl: agent.url, healthUrl: agent.health_check_url }, 'health_check_url fallback failed'); @@ -314,11 +322,12 @@ export class HealthChecker { } } - private async tryA2A(agent: Agent, startTime: number, auth?: SdkAuth): Promise { + private async tryA2A(agent: Agent, startTime: number, auth?: SdkAuth, checkpoint?: () => Promise): Promise { try { // Check for A2A agent card at /.well-known/agent.json const agentCardUrl = `${agent.url.replace(/\/$/, "")}/.well-known/agent.json`; const headers = healthProbeHeaders(auth); + await checkpoint?.(); const response = await fetch(agentCardUrl, { headers, signal: AbortSignal.timeout(5000), @@ -349,6 +358,7 @@ export class HealthChecker { tools_count: toolsCount, }; } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; return { online: false, checked_at: new Date().toISOString(), @@ -357,18 +367,19 @@ export class HealthChecker { } } - async getStats(agent: Agent, auth?: SdkAuth, forceRefresh = false): Promise { + async getStats(agent: Agent, auth?: SdkAuth, forceRefresh = false, checkpoint?: () => Promise): Promise { if (!auth && !forceRefresh) { const cached = this.statsCache.get(agent.url); if (cached) return cached; } - const stats = await this.fetchStats(agent, auth, forceRefresh); + const stats = await this.fetchStats(agent, auth, forceRefresh, checkpoint); + await checkpoint?.(); if (!auth) this.statsCache.set(agent.url, stats); return stats; } - private async fetchStats(agent: Agent, auth?: SdkAuth, forceRefresh = false): Promise { + private async fetchStats(agent: Agent, auth?: SdkAuth, forceRefresh = false, checkpoint?: () => Promise): Promise { const stats: AgentStats = {}; try { @@ -385,15 +396,18 @@ export class HealthChecker { } else if (agent.type === "creative") { // For creative agents, get format count from FormatsService try { + await checkpoint?.(); const formatsProfile = await this.formatsService.getFormatsForAgent(agent, auth, forceRefresh); if (formatsProfile.formats && formatsProfile.formats.length > 0) { stats.creative_formats = formatsProfile.formats.length; } - } catch { + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; // Creative format listing failed } } - } catch { + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; // Stats are optional, failure is ok } diff --git a/server/src/routes/helpers/resolve-user-agent-auth.ts b/server/src/routes/helpers/resolve-user-agent-auth.ts index 463da12b8ac..145b744f18a 100644 --- a/server/src/routes/helpers/resolve-user-agent-auth.ts +++ b/server/src/routes/helpers/resolve-user-agent-auth.ts @@ -13,6 +13,7 @@ import type { AgentContextDatabase } from '../../db/agent-context-db.js'; import { canonicalizeAgentUrl } from '../../db/publisher-db.js'; +import { isComplianceRefreshAccessFailure } from '../../services/compliance-refresh-authorization.js'; import { decodeBasicCredentials, type ResolvedOwnerAuth, @@ -27,6 +28,7 @@ export async function resolveUserAgentAuth( orgId: string, agentUrl: string, logger: WarnLogger, + checkpoint?: () => Promise, ): Promise { const canonicalUrl = canonicalizeAgentUrl(agentUrl); if (!canonicalUrl) { @@ -37,7 +39,9 @@ export async function resolveUserAgentAuth( // Static token lookup throwing falls through to the OAuth branch below — // the connect-form token and the OAuth token are independent paths. try { + await checkpoint?.(); const staticAuth = await agentContextDb.getAuthInfoByOrgAndUrl(orgId, canonicalUrl); + await checkpoint?.(); if (staticAuth) { if (staticAuth.authType === 'basic') { const basic = decodeBasicCredentials(staticAuth.token); @@ -51,15 +55,18 @@ export async function resolveUserAgentAuth( } } } catch (err) { + if (isComplianceRefreshAccessFailure(err)) throw err; logger.warn({ err, agentUrl, orgId }, 'resolveUserAgentAuth: static token lookup failed'); } try { const context = await agentContextDb.getByOrgAndUrl(orgId, canonicalUrl); + await checkpoint?.(); if (!context) return undefined; if (context.has_oauth_token) { const tokens = await agentContextDb.getOAuthTokensByOrgAndUrl(orgId, canonicalUrl); + await checkpoint?.(); if (tokens?.access_token) { if (!tokens.refresh_token) { return { type: 'bearer', token: tokens.access_token }; @@ -75,6 +82,7 @@ export async function resolveUserAgentAuth( }; const client = await agentContextDb.getOAuthClient(context.id); + await checkpoint?.(); if (client) { oauth.client = { client_id: client.client_id, @@ -87,11 +95,13 @@ export async function resolveUserAgentAuth( if (context.has_oauth_client_credentials) { const creds = await agentContextDb.getOAuthClientCredentialsByOrgAndUrl(orgId, canonicalUrl); + await checkpoint?.(); if (creds) return { type: 'oauth_client_credentials', credentials: creds }; } return undefined; } catch (err) { + if (isComplianceRefreshAccessFailure(err)) throw err; logger.warn({ err, agentUrl, orgId }, 'resolveUserAgentAuth: OAuth token lookup failed'); return undefined; } diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index d04bdb39400..9c4e60bcfc3 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -214,7 +214,16 @@ import { ComplianceRefreshRateLimitError, type ClaimedComplianceRefreshRequest, } from "../db/compliance-refresh-requests-db.js"; -import { ComplianceRefreshQueue } from "../services/compliance-refresh-queue.js"; +import { ComplianceRefreshQueue, type ComplianceRefreshExecutionLease } from "../services/compliance-refresh-queue.js"; +import { + captureComplianceRefreshAuthorization, + createComplianceRefreshAuthorizationGuard, + isComplianceRefreshAccessFailure, + ComplianceRefreshAuthorizationError, + isRefreshAdmin, + isRefreshOwner, + resolveRefreshOwnerOrg, +} from "../services/compliance-refresh-authorization.js"; const RegistryAdminAuthorizationUnavailableSchema = z.object({ error: z.literal('admin_authorization_unavailable'), @@ -8288,7 +8297,9 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { function publicRefreshFailure(code: string | null): { code: string; message: string } { switch (code) { case 'authorization_provenance_missing': - return { code, message: 'Refresh requester authorization provenance is unavailable' }; + return { code, message: 'Authenticated credential provenance is missing; submit a new refresh' }; + case 'authorization_unavailable': + return { code, message: 'Refresh authorization is temporarily unavailable' }; case 'authorization_revoked': return { code: 'authorization_revoked', message: 'Access changed before the refresh started' }; case 'monitoring_paused': @@ -8308,6 +8319,14 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } } + function refreshFailureStatus(code: string): number { + if (code === 'authorization_revoked' || code === 'authorization_provenance_missing') return 403; + if (code === 'authorization_unavailable') return 503; + if (code === 'monitoring_paused') return 409; + if (code === 'probe_failed') return 502; + return 500; + } + function prefersAsync(req: Request): boolean { return (req.get('Prefer') ?? '') .split(',') @@ -8327,29 +8346,14 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { async function executeComplianceRefresh( request: ClaimedComplianceRefreshRequest, - lease: { assertValid(): void }, + lease: ComplianceRefreshExecutionLease, ): Promise> { - // Legacy rows cannot establish which WorkOS credential authenticated or - // its authorization epoch. Never infer either from canonical attribution, - // even for an owner, a current admin, or an already checkpointed run. - // #7457/591 alone may enable execution with proven durable provenance. - if (request.requester_type === 'user') { - throw refreshFailure('authorization_provenance_missing', - 'Refresh requester authorization provenance is unavailable'); - } const agentUrl = request.agent_url; - // Preserve the existing additional ownership constraint for any legacy - // static-admin row carrying an owner context. User rows never reach it. - if (request.triggered_by === 'owner_test') { - if ( - !request.owner_org_id - || !request.requested_by_user_id - || !(await isOrgOwnerOfAgent(request.owner_org_id, request.requested_by_user_id, agentUrl)) - ) { - throw refreshFailure('authorization_revoked', 'Agent ownership changed before the refresh started'); - } - } + const authorization = createComplianceRefreshAuthorizationGuard(request, lease); + lease.setCompletionGuard(client => authorization.beforeWrite(client, agentUrl)); + const complianceDb = new ComplianceDatabase(authorization.beforeWrite); + await authorization.checkpoint(); // A prior attempt may have persisted the canonical run and then died // before completing the queue row. Recover that immutable evidence rather @@ -8388,18 +8392,23 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { let complianceAuth: SdkAuth | undefined; if (!persistedRefreshRun) { if (request.owner_org_id) { + await authorization.checkpoint(); const auth = await resolveUserAgentAuth( agentContextDb, request.owner_org_id, agentUrl, logger, + authorization.checkpoint, ); + await authorization.checkpoint(); resolvedAuth = await adaptAuthForSdk(auth, { tokenEndpointLabel: `refresh:${agentUrl}` }); } complianceAuth = resolvedAuth; if (!complianceAuth && !request.owner_org_id) { - const ownerAuth = await complianceDb.resolveOwnerAuth(agentUrl); + await authorization.checkpoint(); + const ownerAuth = await complianceDb.resolveOwnerAuth(agentUrl, authorization.checkpoint); if (ownerAuth) { + await authorization.checkpoint(); complianceAuth = await adaptAuthForSdk(ownerAuth, { tokenEndpointLabel: `admin-refresh:${agentUrl}`, }); @@ -8433,12 +8442,13 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } if (!probeResult) { try { - lease.assertValid(); + await authorization.checkpoint(); probeResult = safeProbeResult(await crawler.refreshSingleAgent(agentUrl, { auth: complianceAuth, + authorization, ...(request.owner_org_id ? { ownerOrgId: request.owner_org_id } : {}), })); - lease.assertValid(); + await authorization.checkpoint(); const probeRecorded = await complianceRefreshQueue.recordProbeResult( request.id, request.lease_token, @@ -8447,7 +8457,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { ); if (!probeRecorded) throw refreshFailure('lease_lost', 'Refresh lease expired'); } catch (error) { - if (error && typeof error === 'object' && 'code' in error && error.code === 'lease_lost') { + if (isComplianceRefreshAccessFailure(error)) { throw error; } const message = error instanceof Error ? error.message : 'Probe failed'; @@ -8513,7 +8523,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { ...complianceSummary, ...badgeEligibilityMetadata(isAuthoritativeComplianceRun(run) ? badgeVersions : []), }; - lease.assertValid(); + await authorization.checkpoint(); if (isAuthoritativeComplianceRun(run)) { if ( Array.isArray(profile.specialisms) @@ -8539,7 +8549,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { }); } } - lease.assertValid(); + await authorization.checkpoint(); } else if (!probeResult.error && !probeResult.oauth_required) { const complianceStart = Date.now(); try { @@ -8555,6 +8565,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { ...(complianceAuth && { auth: complianceAuth }), }; const seededSupportedVersions = await complianceDb.getLastKnownSupportedVersions(agentUrl); + await authorization.checkpoint(); const runTargetSelection = await selectComplianceTargetForAgentSelection( agentUrl, complyOptions, @@ -8566,9 +8577,9 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { throw new Error(UNRESOLVED_COMPLIANCE_TARGET_MESSAGE); } const runTarget = runTargetSelection.target; - lease.assertValid(); + await authorization.checkpoint(); const complyResult = await comply(agentUrl, complyOptions, runTarget); - lease.assertValid(); + await authorization.checkpoint(); if (!selectedComplianceTargetMatchesObservedProfile(runTargetSelection, complyResult.agent_profile)) { throw new Error(UNRESOLVED_COMPLIANCE_TARGET_MESSAGE); } @@ -8611,6 +8622,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { }); } lease.assertValid(); + await authorization.checkpoint(); const { run, storyboardStatuses, replayedExisting } = await complianceDb.recordComplianceRun(dbInput); const passing = storyboardStatuses.filter(status => status.status === 'passing').length; complianceSummary = { @@ -8638,7 +8650,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { && runBadgeEligibleVersions.length > 0 ) { try { - lease.assertValid(); + await authorization.checkpoint(); await runBadgeFanOut({ complianceDb, agentUrl, @@ -8649,12 +8661,13 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { ?? runTargetSelection.supportedVersions, throwOnFailure: true, }); - } catch { + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; throw refreshFailure('badge_update_failed', 'Badge state could not be updated'); } } else { try { - lease.assertValid(); + await authorization.checkpoint(); await revokeUnsupportedPublicBadges({ complianceDb, agentUrl, @@ -8662,7 +8675,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { ?? runTargetSelection.supportedVersions, sourceRunId: run.id, }); - } catch { + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; throw refreshFailure('badge_update_failed', 'Badge state could not be updated'); } } @@ -8670,8 +8684,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } } catch (error) { if ( - error && typeof error === 'object' && 'code' in error - && (error.code === 'badge_update_failed' || error.code === 'lease_lost') + isComplianceRefreshAccessFailure(error) + || (error && typeof error === 'object' && 'code' in error && error.code === 'badge_update_failed') ) { throw error; } @@ -8696,7 +8710,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } } - lease.assertValid(); + await authorization.checkpoint(); return { online: probeResult.online, tools_count: probeResult.tools_count, @@ -8735,46 +8749,47 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { return res.status(401).json({ error: "Authentication required" }); } + if (principal.staticAdmin) { + return res.status(403).json({ + error: 'Queued refresh requires an authenticated user credential', + code: 'authorization_provenance_missing', + }); + } + const orgSelection = parseRequestedOrganizationId(req.body?.organization_id); if (!orgSelection.ok) { return res.status(400).json({ error: "organization_id must be a non-empty organization ID" }); } - const ownerOrgId = principal.user && !principal.staticAdmin ? await resolveOwnerOrgForUser( - principal.user.id, + if (!principal.user) { + return res.status(401).json({ error: "Authentication required" }); + } + const ownerOrgId = await resolveRefreshOwnerOrg( + principal.user, agentUrl, orgSelection.organizationId, - ) : null; - - const isStaticAdmin = principal.staticAdmin; - const isOwner = ownerOrgId !== null; - if (!isOwner && !(await isRegistryAdminRequest(principal))) { - return res.status(403).json({ error: "You do not have permission to refresh this agent" }); - } - - // #7457 owns durable credential + epoch provenance. This schema cannot - // prove any human request, including one currently appearing unlinked. - // Do not enqueue work that the worker must deterministically reject. - // There is deliberately no environment switch to reopen this path. - if (!isStaticAdmin) { - logger.warn({ workosUserId: principal.user?.id, code: 'refresh_authorization_provenance_required' }, - 'Refresh admission fenced until durable authorization provenance is supported'); - res.setHeader('Retry-After', '60'); - res.setHeader('Cache-Control', 'private, no-store'); - return res.status(503).json({ - error: 'Refresh is temporarily unavailable for this credential. Please try again later.', - code: 'refresh_authorization_provenance_required', - retry_after: 60, - }); - } + ); + + const credentialId = principal.user.id; + const context = { + agent_url: agentUrl, + owner_org_id: ownerOrgId, + requester_type: 'user' as const, + requested_by_user_id: credentialId, + requested_by_auth_workos_user_id: credentialId, + triggered_by: ownerOrgId ? 'owner_test' as const : 'manual' as const, + }; + const authorizationFingerprint = await captureComplianceRefreshAuthorization(context); try { const operationId = randomUUID(); const { request, coalesced } = await complianceRefreshQueue.enqueue({ id: operationId, agentUrl, - ownerOrgId: null, - requesterType: 'static_admin', - requestedByUserId: null, - triggeredBy: 'manual', + ownerOrgId, + requesterType: 'user', + requestedByUserId: credentialId, + requestedByAuthWorkosUserId: credentialId, + authorizationFingerprint, + triggeredBy: ownerOrgId ? 'owner_test' : 'manual', agentWindowMs: REFRESH_AGENT_RATE_LIMIT_MS, requesterWindowMs: REFRESH_USER_WINDOW_MS, requesterLimit: REFRESH_USER_LIMIT, @@ -8793,13 +8808,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } if (coalesced && request.status === 'failed') { const failure = publicRefreshFailure(request.last_error_code); - const status = failure.code === 'authorization_revoked' || failure.code === 'authorization_provenance_missing' - ? 403 - : failure.code === 'monitoring_paused' - ? 409 - : failure.code === 'probe_failed' - ? 502 - : 500; + const status = refreshFailureStatus(failure.code); + if (status === 503) res.setHeader('Retry-After', '5'); return res.status(status).json({ error: failure.message, code: failure.code, @@ -8820,13 +8830,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } if (terminal?.status === 'failed') { const failure = publicRefreshFailure(terminal.last_error_code); - const status = failure.code === 'authorization_revoked' || failure.code === 'authorization_provenance_missing' - ? 403 - : failure.code === 'monitoring_paused' - ? 409 - : failure.code === 'probe_failed' - ? 502 - : 500; + const status = refreshFailureStatus(failure.code); + if (status === 503) res.setHeader('Retry-After', '5'); return res.status(status).json({ error: failure.message, code: failure.code }); } } else if (prefersAsync(req)) { @@ -8871,6 +8876,12 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { } } catch (error) { if (respondToAdminAuthorizationError(error, res)) return; + if (isComplianceRefreshAccessFailure(error) && error.code !== 'lease_lost') { + const failure = publicRefreshFailure(error.code); + const status = refreshFailureStatus(error.code); + if (status === 503) res.setHeader('Retry-After', '5'); + return res.status(status).json({ error: failure.message, code: failure.code }); + } logger.error({ err: error, path: req.path }, "Failed to enqueue agent refresh"); res.setHeader('Retry-After', '5'); res.status(503).json({ @@ -8905,10 +8916,10 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { return res.status(404).json({ error: "Refresh operation not found" }); } const ownsCredentialContext = !!operation.owner_org_id && !!principal.user - && await isOrgOwnerOfAgent(operation.owner_org_id, principal.user.id, agentUrl); + && await isRefreshOwner(principal.user, operation.owner_org_id, agentUrl); const canRead = principal.staticAdmin || ownsCredentialContext - || await isRegistryAdminRequest(principal); + || (!!principal.user && await isRefreshAdmin(principal.user)); if (!canRead) { return res.status(404).json({ error: "Refresh operation not found" }); } @@ -8934,6 +8945,12 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { }); } catch (error) { if (respondToAdminAuthorizationError(error, res)) return; + if (error instanceof ComplianceRefreshAuthorizationError) { + const failure = publicRefreshFailure(error.code); + const status = refreshFailureStatus(error.code); + if (status === 503) res.setHeader('Retry-After', '5'); + return res.status(status).json({ error: failure.message, code: failure.code }); + } logger.error({ err: error, path: req.path }, "Failed to read agent refresh operation"); res.setHeader('Retry-After', '5'); return res.status(503).json({ diff --git a/server/src/services/agent-ownership.ts b/server/src/services/agent-ownership.ts index 06b5cfdee91..7e940612576 100644 --- a/server/src/services/agent-ownership.ts +++ b/server/src/services/agent-ownership.ts @@ -100,6 +100,7 @@ export async function findOwnerOrgForUser( export async function findSoleOwnerOrgForUser( userId: string, agentUrl: string, + options: { throwOnError?: boolean } = {}, ): Promise { try { const lookupAgentUrl = canonicalizeAgentUrl(agentUrl) ?? agentUrl; @@ -116,7 +117,8 @@ export async function findSoleOwnerOrgForUser( return result.rows.length === 1 ? result.rows[0].workos_organization_id : null; - } catch { + } catch (error) { + if (options.throwOnError) throw error; return null; } } @@ -135,6 +137,7 @@ export async function isOrgOwnerOfAgent( orgId: string, userId: string, agentUrl: string, + options: { throwOnError?: boolean } = {}, ): Promise { try { const lookupAgentUrl = canonicalizeAgentUrl(agentUrl) ?? agentUrl; @@ -149,7 +152,8 @@ export async function isOrgOwnerOfAgent( [orgId, JSON.stringify([{ url: lookupAgentUrl }]), userId], ); return result.rows.length > 0; - } catch { + } catch (error) { + if (options.throwOnError) throw error; return false; } } @@ -196,12 +200,13 @@ export async function resolveOwnerOrgForUser( userId: string, agentUrl: string, requestedOrgId?: string, + options: { throwOnError?: boolean } = {}, ): Promise { if (requestedOrgId === undefined) { - return findSoleOwnerOrgForUser(userId, agentUrl); + return findSoleOwnerOrgForUser(userId, agentUrl, options); } - return (await isOrgOwnerOfAgent(requestedOrgId, userId, agentUrl)) + return (await isOrgOwnerOfAgent(requestedOrgId, userId, agentUrl, options)) ? requestedOrgId : null; } diff --git a/server/src/services/badge-issuance.ts b/server/src/services/badge-issuance.ts index 8e683a4fd93..c3c35822362 100644 --- a/server/src/services/badge-issuance.ts +++ b/server/src/services/badge-issuance.ts @@ -13,6 +13,7 @@ import { query } from '../db/client.js'; import { notifySystemError } from '../addie/error-notifier.js'; import { logger as baseLogger } from '../logger.js'; import { getEffectiveGradingDecision } from '../db/verification-profile-db.js'; +import { isComplianceRefreshAccessFailure } from './compliance-refresh-authorization.js'; const logger = baseLogger.child({ module: 'badge-issuance' }); @@ -584,6 +585,7 @@ export async function runBadgeFanOut(params: { for (const degraded of versionResult.degraded) aggregate.degraded.push(degraded); for (const unchanged of versionResult.unchanged) aggregate.unchanged.push(unchanged); } catch (versionError) { + if (isComplianceRefreshAccessFailure(versionError)) throw versionError; processingFailed = true; const errorMessage = versionError instanceof Error ? versionError.message : String(versionError); logger.error( diff --git a/server/src/services/compliance-refresh-authorization.ts b/server/src/services/compliance-refresh-authorization.ts new file mode 100644 index 00000000000..e422e55fdad --- /dev/null +++ b/server/src/services/compliance-refresh-authorization.ts @@ -0,0 +1,212 @@ +import type { PoolClient } from 'pg'; +import type { AAOAdminPrincipal } from '../addie/admin-status-lookup.js'; +import { isBreakGlassAdminEmail } from '../auth/admin-access.js'; +import { getAuthorizationEnforcementWorkos } from '../auth/workos-client.js'; +import { getAuthorizationFingerprint } from '../db/authorization-epoch-db.js'; +import { query, withDatabaseDeadline } from '../db/client.js'; +import { + assertComplianceRefreshAuthorizationFingerprint, + ComplianceRefreshLeaseLostError, + ComplianceRefreshProvenanceError, + hasComplianceRefreshProvenance, + type ClaimedComplianceRefreshRequest, + type ComplianceRefreshRequest, +} from '../db/compliance-refresh-requests-db.js'; +import { isOrgOwnerOfAgent, resolveOwnerOrgForUser } from './agent-ownership.js'; + +export class ComplianceRefreshAuthorizationError extends Error { + constructor(readonly code: 'authorization_revoked' | 'authorization_unavailable') { + super(code === 'authorization_revoked' + ? 'Access changed during the refresh' + : 'Refresh authorization is temporarily unavailable'); + this.name = 'ComplianceRefreshAuthorizationError'; + } +} + +export function isComplianceRefreshAccessFailure(error: unknown): error is Error & { code: string } { + return !!error && typeof error === 'object' && 'code' in error + && (error.code === 'authorization_revoked' || error.code === 'authorization_unavailable' + || error.code === 'authorization_provenance_missing' || error.code === 'lease_lost'); +} + +/** Preserve an unavailable authority source instead of turning it into denial. */ +async function lookupAuthority(lookup: () => Promise): Promise { + try { + return await withDatabaseDeadline(Date.now() + 5_000, lookup); + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; + throw new ComplianceRefreshAuthorizationError('authorization_unavailable'); + } +} + +export function resolveRefreshOwnerOrg(principal: AAOAdminPrincipal, agentUrl: string, orgId?: string) { + return lookupAuthority(() => resolveOwnerOrgForUser( + principal.authWorkosUserId ?? principal.id, agentUrl, orgId, { throwOnError: true }, + )); +} + +export function isRefreshOwner(principal: AAOAdminPrincipal, orgId: string, agentUrl: string) { + return lookupAuthority(() => isOrgOwnerOfAgent( + orgId, principal.authWorkosUserId ?? principal.id, agentUrl, { throwOnError: true }, + )); +} + +/** No canonical identity resolver or process-local authorization cache. */ +async function isExactCredentialAdmin(principal: AAOAdminPrincipal): Promise { + try { + return await lookupAuthority(async () => { + const result = await query<{ is_admin: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM working_group_memberships m + WHERE m.working_group_id = g.id AND m.workos_user_id = $1 AND m.status = 'active' + ) AS is_admin FROM working_groups g WHERE g.slug = 'aao-admin'`, + [principal.authWorkosUserId ?? principal.id], + ); + if (!result.rows[0]) throw new ComplianceRefreshAuthorizationError('authorization_unavailable'); + return result.rows[0].is_admin || isBreakGlassAdminEmail(principal.email); + }); + } catch (error) { + // Independent break-glass authority uses only the just-fetched WorkOS email. + if (isBreakGlassAdminEmail(principal.email)) return true; + throw error; + } +} + +async function currentCredential(credentialId: string): Promise { + let credential: { id: string; email: string }; + let timeout: NodeJS.Timeout | undefined; + try { + // Bound the whole lookup, including body decoding (the SDK timer ends at + // headers). A late result cannot resume this authorization. + credential = await Promise.race([ + getAuthorizationEnforcementWorkos().userManagement.getUser(credentialId), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new ComplianceRefreshAuthorizationError('authorization_unavailable')), 5_000); + }), + ]); + } catch (error) { + const deleted = error && typeof error === 'object' && 'status' in error && error.status === 404; + throw new ComplianceRefreshAuthorizationError(deleted ? 'authorization_revoked' : 'authorization_unavailable'); + } finally { + if (timeout) clearTimeout(timeout); + } + if (credential?.id !== credentialId) throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + if (typeof credential.email !== 'string' || !credential.email.trim()) { + throw new ComplianceRefreshAuthorizationError('authorization_unavailable'); + } + return { id: credentialId, authWorkosUserId: credentialId, email: credential.email }; +} + +export async function isRefreshAdmin(principal: AAOAdminPrincipal): Promise { + return isExactCredentialAdmin(await currentCredential(principal.authWorkosUserId ?? principal.id)); +} + +export type ComplianceRefreshAuthorizationContext = Pick; + +async function fingerprint(credentialId: string): Promise { + return lookupAuthority(() => getAuthorizationFingerprint([credentialId])); +} + +async function validateAuthority( + request: ComplianceRefreshAuthorizationContext, + admittedFingerprint: string, +): Promise { + if (!hasComplianceRefreshProvenance({ ...request, authorization_fingerprint: admittedFingerprint })) { + throw new ComplianceRefreshProvenanceError(); + } + const credentialId = request.requested_by_auth_workos_user_id!; + if (await fingerprint(credentialId) !== admittedFingerprint) { + throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + } + const principal = await currentCredential(credentialId); + const authorized = request.triggered_by === 'owner_test' + ? await isRefreshOwner(principal, request.owner_org_id!, request.agent_url) + : await isExactCredentialAdmin(principal); + if (!authorized || await fingerprint(credentialId) !== admittedFingerprint) { + throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + } + return principal; +} + +/** Capture only after proving the exact credential; never backfill old jobs. */ +export async function captureComplianceRefreshAuthorization( + context: ComplianceRefreshAuthorizationContext, +): Promise { + if (!hasComplianceRefreshProvenance({ ...context, authorization_fingerprint: '' })) { + throw new ComplianceRefreshProvenanceError(); + } + const admitted = await fingerprint(context.requested_by_auth_workos_user_id!); + await validateAuthority(context, admitted); + return admitted; +} + +export async function authorizeComplianceRefresh(request: ComplianceRefreshRequest): Promise { + await validateAuthority(request, request.authorization_fingerprint); +} + +/** Runs inside the protected write's transaction, before any mutation. */ +export type ComplianceRefreshWriteGuard = (client: PoolClient, agentUrl: string) => Promise; +export interface ComplianceRefreshAuthorizationGuard { + checkpoint(): Promise; + beforeWrite: ComplianceRefreshWriteGuard; +} + +export function createComplianceRefreshAuthorizationGuard( + request: ClaimedComplianceRefreshRequest, + lease: { assertValid(): void }, +): ComplianceRefreshAuthorizationGuard { + const checkpoint = async () => { + lease.assertValid(); + await authorizeComplianceRefresh(request); + lease.assertValid(); + }; + return { + checkpoint, + beforeWrite: async (client, agentUrl) => { + lease.assertValid(); + if (agentUrl !== request.agent_url) throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + // Remote validation precedes row locks. Local identity/membership + // revocation and the protected write then serialize in one transaction. + const principal = await validateAuthority(request, request.authorization_fingerprint); + try { + await client.query("SELECT set_config('statement_timeout', '5000ms', true)"); + await client.query("SELECT set_config('lock_timeout', '2000ms', true)"); + await assertComplianceRefreshAuthorizationFingerprint( + client, request.requested_by_auth_workos_user_id!, request.authorization_fingerprint, + ); + if (request.triggered_by === 'owner_test') { + const owner = await client.query( + `SELECT om.workos_user_id FROM organization_memberships om + JOIN member_profiles mp ON mp.workos_organization_id = om.workos_organization_id + WHERE om.workos_user_id = $1 AND mp.workos_organization_id = $2 + AND mp.agents @> $3::jsonb FOR SHARE OF om, mp`, + [principal.id, request.owner_org_id, JSON.stringify([{ url: request.agent_url }])], + ); + if (!owner.rowCount) throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + } else if (!isBreakGlassAdminEmail(principal.email)) { + const admin = await client.query( + `SELECT m.id FROM working_group_memberships m + JOIN working_groups g ON g.id = m.working_group_id + WHERE g.slug = 'aao-admin' AND m.workos_user_id = $1 AND m.status = 'active' + FOR SHARE OF m, g`, + [principal.id], + ); + if (!admin.rowCount) throw new ComplianceRefreshAuthorizationError('authorization_revoked'); + } + const operation = await client.query( + `SELECT id FROM agent_compliance_refresh_requests + WHERE id = $1 AND agent_url = $2 AND status = 'running' AND lease_token = $3 + AND lease_expires_at > clock_timestamp() FOR SHARE`, + [request.id, request.agent_url, request.lease_token], + ); + if (!operation.rowCount) throw new ComplianceRefreshLeaseLostError(); + lease.assertValid(); + } catch (error) { + if (isComplianceRefreshAccessFailure(error)) throw error; + throw new ComplianceRefreshAuthorizationError('authorization_unavailable'); + } + }, + }; +} diff --git a/server/src/services/compliance-refresh-queue.ts b/server/src/services/compliance-refresh-queue.ts index 2f575170a6e..51e1afae07d 100644 --- a/server/src/services/compliance-refresh-queue.ts +++ b/server/src/services/compliance-refresh-queue.ts @@ -1,3 +1,4 @@ +import type { PoolClient } from 'pg'; import { createLogger } from '../logger.js'; import { ComplianceRefreshRequestsDatabase, @@ -16,7 +17,9 @@ const RETENTION_MS = 7 * 24 * 60 * 60_000; function stableFailureMessage(code: string): string { switch (code) { case 'authorization_provenance_missing': - return 'Refresh requester authorization provenance is unavailable'; + return 'Authenticated credential provenance is missing; submit a new refresh'; + case 'authorization_unavailable': + return 'Refresh authorization is temporarily unavailable'; case 'authorization_revoked': return 'Authorization changed before the refresh started'; case 'monitoring_paused': @@ -33,6 +36,7 @@ function stableFailureMessage(code: string): string { function stableFailureCode(code: unknown): string { return code === 'authorization_revoked' || code === 'authorization_provenance_missing' + || code === 'authorization_unavailable' || code === 'monitoring_paused' || code === 'probe_failed' || code === 'compliance_failed' @@ -41,9 +45,14 @@ function stableFailureCode(code: unknown): string { : 'refresh_failed'; } +export interface ComplianceRefreshExecutionLease { + assertValid(): void; + setCompletionGuard(guard: (client: PoolClient) => Promise): void; +} + export type ComplianceRefreshExecutor = ( request: ClaimedComplianceRefreshRequest, - lease: { assertValid(): void }, + lease: ComplianceRefreshExecutionLease, ) => Promise>; export class ComplianceRefreshQueue { @@ -172,12 +181,29 @@ export class ComplianceRefreshQueue { throw error; } }; - const result = await this.execute(request, { assertValid }); + let completionGuard: ((client: PoolClient) => Promise) | undefined; + const result = await this.execute(request, { + assertValid, + setCompletionGuard(guard) { + if (completionGuard || typeof guard !== 'function') { + throw Object.assign(new Error('Refresh completion authorization is unavailable'), { + code: 'authorization_unavailable', + }); + } + completionGuard = guard; + }, + }); if (!leaseValid || Date.now() >= leaseExpiresAt) { logger.warn({ operationId: request.id }, 'Compliance refresh completed after losing its lease'); return 'lostLease'; } - const recorded = await this.db.markSucceeded(request.id, request.lease_token, result); + if (!completionGuard) { + throw Object.assign(new Error('Refresh completion authorization is unavailable'), { + code: 'authorization_unavailable', + }); + } + assertValid(); + const recorded = await this.db.markSucceeded(request.id, request.lease_token, result, completionGuard); if (!recorded) return 'lostLease'; logger.info( { operationId: request.id, agentUrl: request.agent_url, attempts: request.attempts }, @@ -185,6 +211,9 @@ export class ComplianceRefreshQueue { ); return 'succeeded'; } catch (error) { + // A timed-out probe may still have an awaited local continuation. Stop + // its next authorization checkpoint before persisting this failure. + leaseValid = false; const rawCode = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string' ? error.code @@ -194,7 +223,7 @@ export class ComplianceRefreshQueue { return 'lostLease'; } const code = stableFailureCode(rawCode); - if (code === 'badge_update_failed') { + if (code === 'badge_update_failed' || code === 'authorization_unavailable') { const requeued = await this.db.requeueAfterFailure( request.id, request.lease_token, @@ -216,6 +245,7 @@ export class ComplianceRefreshQueue { ); return 'failed'; } finally { + leaseValid = false; clearInterval(heartbeat); await executionFence.release(); } diff --git a/server/tests/integration/compliance-refresh-credential-migration.test.ts b/server/tests/integration/compliance-refresh-credential-migration.test.ts new file mode 100644 index 00000000000..4218f008246 --- /dev/null +++ b/server/tests/integration/compliance-refresh-credential-migration.test.ts @@ -0,0 +1,307 @@ +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { Pool, type PoolClient } from 'pg'; +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { closeDatabase, initializeDatabase } from '../../src/db/client.js'; +import { + ComplianceRefreshRequestsDatabase, + type ClaimedComplianceRefreshRequest, +} from '../../src/db/compliance-refresh-requests-db.js'; +import { ComplianceRefreshQueue } from '../../src/services/compliance-refresh-queue.js'; + +const workos = vi.hoisted(() => { + const getUser = vi.fn(); + return { getUser, getClient: vi.fn(() => ({ userManagement: { getUser } })) }; +}); +vi.mock('../../src/auth/workos-client.js', () => ({ getAuthorizationEnforcementWorkos: workos.getClient })); + +import { authorizeComplianceRefresh } from '../../src/services/compliance-refresh-authorization.js'; + +const original = readFileSync(new URL('../../src/db/migrations/571_agent_compliance_refresh_requests.sql', import.meta.url), 'utf8'); +const correction = readFileSync(new URL('../../src/db/migrations/599_compliance_refresh_authenticated_credential.sql', import.meta.url), 'utf8'); +const connectionString = process.env.DATABASE_URL || 'postgresql://adcp:localdev@localhost:5432/adcp_test'; +const admin = new Pool({ connectionString }); +const contract = "SELECT set_config('adcp.compliance_refresh_writer_contract', 'authenticated-credential-v1', true)"; + +// This is the pre-599 claim shape: it deliberately knows nothing about the new +// authenticated provenance fields or the worker-contract setting. +const oldClaim = `UPDATE agent_compliance_refresh_requests + SET status = 'running', attempts = attempts + 1, lease_owner = 'old-worker', + lease_token = gen_random_uuid(), lease_expires_at = NOW() + INTERVAL '1 minute' + WHERE id = $1 RETURNING *`; + +describe('compliance refresh authenticated credential migration', () => { + let schema: string; + let pool: Pool; + let db: ComplianceRefreshRequestsDatabase; + + beforeEach(async () => { + vi.clearAllMocks(); + schema = `refresh_credential_${randomUUID().replaceAll('-', '')}`; + await admin.query(`CREATE SCHEMA ${schema}`); + const scoped = new URL(connectionString); + scoped.searchParams.set('options', `-c search_path=${schema}`); + pool = initializeDatabase({ connectionString: scoped.toString() }); + await pool.query('CREATE TABLE agent_compliance_runs (id UUID PRIMARY KEY)'); + await pool.query('CREATE TABLE users (workos_user_id TEXT PRIMARY KEY, email TEXT NOT NULL)'); + await pool.query(`CREATE TABLE authorization_epochs ( + workos_user_id TEXT PRIMARY KEY REFERENCES users(workos_user_id) ON DELETE CASCADE, + epoch BIGINT NOT NULL DEFAULT 1 + )`); + await pool.query("INSERT INTO users VALUES ('user-exact', 'exact@example.test')"); + await pool.query(original); + db = new ComplianceRefreshRequestsDatabase(); + }); + + afterEach(async () => { + await closeDatabase(); + await admin.query(`DROP SCHEMA ${schema} CASCADE`); + }); + + afterAll(async () => { await admin.end(); }); + + async function legacy(status: 'queued' | 'running' | 'succeeded' | 'failed', requesterType = 'user') { + const id = randomUUID(); + const agentUrl = `https://${id}.example.test/mcp`; + await pool.query(`INSERT INTO agent_compliance_refresh_requests + (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, test_session_id, + status, lease_owner, lease_token, lease_expires_at, result_json, completed_at) + VALUES ($1, $2, NULL, $3, $4, 'manual', $5, $6, + CASE WHEN $6 = 'running' THEN 'old-worker' END, + CASE WHEN $6 = 'running' THEN gen_random_uuid() END, + CASE WHEN $6 = 'running' THEN NOW() + INTERVAL '1 minute' END, + CASE WHEN $6 = 'succeeded' THEN '{"online":true}'::jsonb END, + CASE WHEN $6 IN ('succeeded', 'failed') THEN NOW() END)`, + [id, agentUrl, requesterType, requesterType === 'user' ? 'user-canonical-linked' : null, `legacy-${id}`, status]); + return { id, agentUrl }; + } + + async function migrate(client?: PoolClient): Promise { + const own = client ?? await pool.connect(); + try { + await own.query('BEGIN'); + await own.query(correction); + await own.query('COMMIT'); + } catch (error) { + await own.query('ROLLBACK'); + throw error; + } finally { + if (!client) own.release(); + } + } + + async function enqueue(agentUrl = `https://${randomUUID()}.example.test/mcp`) { + return db.createOrGetActive({ id: randomUUID(), agentUrl, ownerOrgId: null, requesterType: 'user', + requestedByUserId: 'user-exact', requestedByAuthWorkosUserId: 'user-exact', + authorizationFingerprint: '', triggeredBy: 'manual', requesterLimit: 1 }); + } + + it('marks real legacy histories unproven, rejects pending jobs, and allows fresh resubmission', async () => { + const queued = await legacy('queued'); + const staticQueued = await legacy('queued', 'static_admin'); + const succeeded = await legacy('succeeded'); + const failed = await legacy('failed'); + const historical = (await pool.query('SELECT * FROM agent_compliance_refresh_requests WHERE id = $1', [succeeded.id])).rows[0]; + await migrate(); + + const rows = (await pool.query('SELECT * FROM agent_compliance_refresh_requests')).rows; + for (const row of rows) { + expect(row.requested_by_auth_workos_user_id).toBe('__unproven__'); + expect(row.authorization_fingerprint).toBe('__unproven__'); + expect(row.requested_by_user_id).toBe(row.requester_type === 'user' ? 'user-canonical-linked' : null); + } + for (const pending of [queued, staticQueued]) { + expect(rows.find(row => row.id === pending.id)).toMatchObject({ status: 'failed', + last_error_code: 'authorization_provenance_missing', lease_token: null, result_json: null }); + } + expect(rows.find(row => row.id === succeeded.id)).toMatchObject({ status: 'succeeded', + result_json: historical.result_json, completed_at: historical.completed_at }); + expect(rows.find(row => row.id === failed.id)?.status).toBe('failed'); + const fresh = await enqueue(queued.agentUrl); + expect(fresh.coalesced).toBe(false); + expect(fresh.request.id).not.toBe(queued.id); + expect(fresh.request.authorization_fingerprint).toBe(''); + const shape = await pool.query(`SELECT column_name, is_nullable, column_default + FROM information_schema.columns WHERE table_schema = $1 + AND table_name = 'agent_compliance_refresh_requests' + AND column_name IN ('requested_by_auth_workos_user_id', 'authorization_fingerprint')`, [schema]); + expect(shape.rows).toHaveLength(2); + for (const column of shape.rows) expect(column).toMatchObject({ is_nullable: 'NO', column_default: null }); + }); + + it('blocks actual workers and authorization for migrated queued user and static-admin jobs', async () => { + const pending = [await legacy('queued'), await legacy('queued', 'static_admin')]; + await migrate(); + expect(await db.claimDue('new-worker', 2, 60_000)).toEqual({ requests: [], terminalizedExpired: 0 }); + + const resolveSecrets = vi.fn(); + const executor = vi.fn(async (request: ClaimedComplianceRefreshRequest) => { + await authorizeComplianceRefresh(request); + await resolveSecrets(); + return { online: true }; + }); + const queue = new ComplianceRefreshQueue(executor, db, 'migration-proof-worker'); + expect(await queue.processQueue()).toEqual({ claimed: 0, succeeded: 0, failed: 0, lostLease: 0 }); + expect(executor).not.toHaveBeenCalled(); + expect(resolveSecrets).not.toHaveBeenCalled(); + + for (const old of pending) { + const migrated = (await db.getById(old.id))!; + await expect(authorizeComplianceRefresh(migrated)).rejects.toMatchObject({ + code: 'authorization_provenance_missing', + message: 'Authenticated credential provenance is missing; submit a new refresh', + }); + expect(migrated).toMatchObject({ status: 'failed', requested_by_auth_workos_user_id: '__unproven__', + authorization_fingerprint: '__unproven__', last_error_code: 'authorization_provenance_missing' }); + } + expect(workos.getClient).not.toHaveBeenCalled(); + expect(workos.getUser).not.toHaveBeenCalled(); + + const fresh = await enqueue(pending[0].agentUrl); + expect(fresh.coalesced).toBe(false); + expect(pending.map(old => old.id)).not.toContain(fresh.request.id); + const claims = await db.claimDue('new-worker', 2, 60_000); + expect(claims.requests.map(request => request.id)).toEqual([fresh.request.id]); + }); + + it('rejects resurrecting either sentinel job even with the valid new-worker contract', async () => { + const pending = [await legacy('queued'), await legacy('queued', 'static_admin')]; + await migrate(); + const client = await pool.connect(); + try { + for (const old of pending) { + for (const status of ['queued', 'running']) { + await client.query('BEGIN'); + await client.query(contract); + // Satisfy every original 571 lease/result/completion shape. The + // rejection must come from the new provenance CHECK itself. + await expect(client.query(`UPDATE agent_compliance_refresh_requests + SET status = $2, completed_at = NULL, result_json = NULL, + lease_owner = CASE WHEN $2 = 'running' THEN 'new-worker' END, + lease_token = CASE WHEN $2 = 'running' THEN gen_random_uuid() END, + lease_expires_at = CASE WHEN $2 = 'running' THEN NOW() + INTERVAL '1 minute' END + WHERE id = $1`, [old.id, status])).rejects.toMatchObject({ + code: '23514', constraint: 'agent_compliance_refresh_authenticated_credential', + }); + await client.query('ROLLBACK'); + expect((await db.getById(old.id))?.status).toBe('failed'); + } + } + } finally { + await client.query('ROLLBACK'); + client.release(); + } + expect((await db.claimDue('new-worker', 2, 60_000)).requests).toEqual([]); + }); + + it.each([false, true])('rejects every running legacy row (expired: %s) without partial schema or history edits', async expired => { + const queued = await legacy('queued'); + const running = await legacy('running'); + if (expired) await pool.query("UPDATE agent_compliance_refresh_requests SET lease_expires_at = NOW() - INTERVAL '1 hour', attempts = max_attempts WHERE id = $1", [running.id]); + await expect(migrate()).rejects.toThrow('Drain all running'); + expect((await pool.query('SELECT status FROM agent_compliance_refresh_requests WHERE id = $1', [queued.id])).rows[0].status) + .toBe('queued'); + const columns = await pool.query(`SELECT column_name FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 'agent_compliance_refresh_requests' + AND column_name = 'requested_by_auth_workos_user_id'`, [schema]); + expect(columns.rows).toHaveLength(0); + }); + + it('checks running claims only after waiting for a concurrent old claim transaction', async () => { + const pending = await legacy('queued'); + const oldWorker = await pool.connect(); + const migrator = await pool.connect(); + const migratorPid = (await migrator.query('SELECT pg_backend_pid() AS pid')).rows[0].pid; + await oldWorker.query('BEGIN'); + await oldWorker.query(oldClaim, [pending.id]); + const migration = migrate(migrator).then(() => ({ passed: true }), error => ({ error })); + try { + await vi.waitFor(async () => { + const waiting = await admin.query("SELECT wait_event_type FROM pg_stat_activity WHERE pid = $1", [migratorPid]); + expect(waiting.rows[0].wait_event_type).toBe('Lock'); + }); + await oldWorker.query('COMMIT'); + expect(await migration).toMatchObject({ error: { message: expect.stringContaining('Drain all running') } }); + } finally { + await oldWorker.query('ROLLBACK'); + await migration; + oldWorker.release(); + migrator.release(); + } + }); + + it('rejects old inserts, malformed fingerprints, static admissions, and direct running inserts', async () => { + await migrate(); + await expect(legacy('queued')).rejects.toMatchObject({ code: '23502' }); + await expect(legacy('queued', 'static_admin')).rejects.toThrow('authenticated user provenance'); + const insert = (credential: string | null, fingerprint: string | null, requesterType = 'user', status = 'queued') => { + const id = randomUUID(); + return pool.query(`INSERT INTO agent_compliance_refresh_requests + (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, test_session_id, + requested_by_auth_workos_user_id, authorization_fingerprint, status) + VALUES ($1, $2, NULL, $3, $4, 'manual', $5, $6, $7, $8)`, + [id, `https://${id}.example.test/mcp`, requesterType, requesterType === 'user' ? 'user-exact' : null, + `new-${id}`, credential, fingerprint, status]); + }; + for (const [credential, fingerprint] of [[null, null], [null, ''], ['user-exact', null]]) { + await expect(insert(credential, fingerprint)).rejects.toMatchObject({ code: '23502' }); + } + expect((await pool.query('SELECT COUNT(*)::integer AS count FROM agent_compliance_refresh_requests')).rows[0].count).toBe(0); + expect((await db.claimDue('new-worker', 2, 60_000)).requests).toEqual([]); + for (const [credential, fingerprint] of [ + ['', ''], [' user-exact ', ''], ['user-linked', ''], + ['__unproven__', '__unproven__'], ['user-exact', '__unproven__'], ['user-exact', 'user-linked:1'], + ['user-exact', 'user-exact:0'], ['user-exact', 'user-exact:01'], ['user-exact', 'user-exact:9223372036854775808'], + ]) await expect(insert(credential, fingerprint)).rejects.toBeDefined(); + await expect(insert('user-exact', '', 'static_admin')).rejects.toThrow('authenticated user provenance'); + await expect(insert('user-exact', '', 'user', 'running')).rejects.toThrow('authenticated user provenance'); + await insert('user-exact', 'user-exact:9223372036854775807'); + }); + + it('blocks old claims and reclaims while preserving new lease lifecycle writes', async () => { + await migrate(); + const created = await enqueue(); + await expect(pool.query(oldClaim, [created.request.id])).rejects.toThrow('worker contract'); + expect((await db.getById(created.request.id))?.status).toBe('queued'); + const [claim] = (await db.claimDue('new-worker', 1, 60_000)).requests; + await expect(pool.query(oldClaim, [claim.id])).rejects.toThrow('worker contract'); + expect((await db.getById(claim.id))?.lease_token).toBe(claim.lease_token); + expect(await db.heartbeat(claim.id, claim.lease_token, 60_000)).toBe(true); + expect(await db.recordProbeResult(claim.id, claim.lease_token, { online: true }, false)).toBe(true); + expect(await db.deferClaim(claim.id, claim.lease_token, 0)).toBe(true); + const [reclaimed] = (await db.claimDue('new-worker-2', 1, 60_000)).requests; + expect(await db.markFailed(reclaimed.id, reclaimed.lease_token, 'authorization_revoked', 'Revoked')).toBe(true); + }); + + it.each(['COMMIT', 'ROLLBACK'])('does not leak its claim protocol setting after %s', async end => { + await migrate(); + const created = await enqueue(); + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query(contract); + await client.query(oldClaim, [created.request.id]); + await client.query(end); + await expect(client.query(oldClaim, [created.request.id])).rejects.toThrow('worker contract'); + } finally { + await client.query('ROLLBACK'); + client.release(); + } + }); + + it('keeps credential, fingerprint, and context immutable, including unproven history', async () => { + const old = await legacy('succeeded'); + await migrate(); + const created = await enqueue(); + for (const assignment of ["requested_by_auth_workos_user_id = 'user-linked'", "authorization_fingerprint = 'user-exact:1'", + "requested_by_user_id = 'user-linked'", "agent_url = 'https://changed.example.test/mcp'", "owner_org_id = 'org-other'"]) { + await expect(pool.query(`UPDATE agent_compliance_refresh_requests SET ${assignment} WHERE id = $1`, [created.request.id])) + .rejects.toThrow('immutable'); + } + await expect(pool.query(`UPDATE agent_compliance_refresh_requests + SET requested_by_auth_workos_user_id = 'user-canonical-linked', authorization_fingerprint = '' WHERE id = $1`, [old.id])) + .rejects.toThrow('immutable'); + await expect(pool.query(oldClaim, [old.id])).rejects.toThrow('worker contract'); + expect((await db.getById(old.id))?.status).toBe('succeeded'); + }); +}); diff --git a/server/tests/integration/compliance-refresh-requests-queue.test.ts b/server/tests/integration/compliance-refresh-requests-queue.test.ts index b3cbe4318f6..cf5358f095d 100644 --- a/server/tests/integration/compliance-refresh-requests-queue.test.ts +++ b/server/tests/integration/compliance-refresh-requests-queue.test.ts @@ -1,10 +1,12 @@ import { randomUUID } from 'node:crypto'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Pool } from 'pg'; import { closeDatabase, initializeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; import { + assertComplianceRefreshAuthorizationFingerprint, ComplianceRefreshInProgressError, + ComplianceRefreshProvenanceError, ComplianceRefreshRateLimitError, ComplianceRefreshRequestsDatabase, } from '../../src/db/compliance-refresh-requests-db.js'; @@ -23,7 +25,11 @@ describe('durable compliance refresh request queue', () => { }); beforeEach(async () => { - if (ids.length === 0) return; + await pool.query( + `INSERT INTO users (workos_user_id, email) VALUES ('user-refresh-test', 'refresh@example.test'), ('user-other', 'other@example.test') + ON CONFLICT (workos_user_id) DO NOTHING`, + ); + await pool.query("DELETE FROM authorization_epochs WHERE workos_user_id IN ('user-refresh-test', 'user-other')"); await pool.query('DELETE FROM agent_compliance_refresh_requests WHERE id = ANY($1::uuid[])', [[...ids]]); ids.length = 0; }); @@ -32,6 +38,7 @@ describe('durable compliance refresh request queue', () => { if (ids.length > 0) { await pool.query('DELETE FROM agent_compliance_refresh_requests WHERE id = ANY($1::uuid[])', [[...ids]]); } + await pool.query("DELETE FROM users WHERE workos_user_id IN ('user-refresh-test', 'user-other')"); await closeDatabase(); }); @@ -44,11 +51,20 @@ describe('durable compliance refresh request queue', () => { ownerOrgId: 'org-refresh-test', requesterType: 'user', requestedByUserId: 'user-refresh-test', + requestedByAuthWorkosUserId: 'user-refresh-test', + authorizationFingerprint: '', triggeredBy: 'owner_test', ...overrides, }); } + async function complete(id: string, leaseToken: string, result: Record) { + const request = (await db.getById(id))!; + return db.markSucceeded(id, leaseToken, result, client => assertComplianceRefreshAuthorizationFingerprint( + client, request.requested_by_auth_workos_user_id, request.authorization_fingerprint, + )); + } + it('persists a stable test-session identity without credential material', async () => { const created = await enqueue(); const stored = await db.getById(created.request.id); @@ -56,6 +72,8 @@ describe('durable compliance refresh request queue', () => { expect(stored).toMatchObject({ status: 'queued', owner_org_id: 'org-refresh-test', + requested_by_auth_workos_user_id: 'user-refresh-test', + authorization_fingerprint: '', test_session_id: `owner-refresh-${created.request.id}`, result_json: null, }); @@ -77,7 +95,9 @@ describe('durable compliance refresh request queue', () => { agentUrl, ownerOrgId: 'org-refresh-test', requesterType: 'user', - requestedByUserId: 'another-user-in-org', + requestedByUserId: 'user-refresh-test', + requestedByAuthWorkosUserId: 'user-refresh-test', + authorizationFingerprint: '', triggeredBy: 'owner_test', }); @@ -90,16 +110,44 @@ describe('durable compliance refresh request queue', () => { await enqueue({ agentUrl }); for (const context of [ - { ownerOrgId: 'org-other', requesterType: 'user' as const, requestedByUserId: 'user-other', triggeredBy: 'owner_test' as const }, - { ownerOrgId: null, requesterType: 'static_admin' as const, requestedByUserId: null, triggeredBy: 'manual' as const }, + { ownerOrgId: 'org-other', requesterType: 'user' as const, requestedByUserId: 'user-other', requestedByAuthWorkosUserId: 'user-other', triggeredBy: 'owner_test' as const }, + { ownerOrgId: 'org-refresh-test', requesterType: 'user' as const, requestedByUserId: 'user-other', requestedByAuthWorkosUserId: 'user-other', triggeredBy: 'owner_test' as const }, ]) { const id = randomUUID(); ids.push(id); - await expect(db.createOrGetActive({ id, agentUrl, ...context })) + await expect(db.createOrGetActive({ id, agentUrl, authorizationFingerprint: '', ...context })) .rejects.toBeInstanceOf(ComplianceRefreshInProgressError); } }); + it.each(['queued', 'succeeded'] as const)('rejects malformed caller provenance before coalescing a %s operation', async (status) => { + const first = await enqueue(); + if (status === 'succeeded') { + const [claimed] = (await db.claimDue('worker-test', 1, 60_000)).requests; + await complete(claimed.id, claimed.lease_token, { online: true }); + } + for (const credential of [undefined, null, '', ' ', ' user-refresh-test ', 'user-linked']) { + await expect(enqueue({ + agentUrl: first.request.agent_url, + requestedByAuthWorkosUserId: credential, + })).rejects.toBeInstanceOf(ComplianceRefreshProvenanceError); + } + const duplicate = await enqueue({ agentUrl: first.request.agent_url }); + expect(duplicate.coalesced).toBe(true); + expect(duplicate.request.id).toBe(first.request.id); + }); + + it('does not coalesce a recent terminal operation from another authenticated credential', async () => { + const first = await enqueue(); + const [claimed] = (await db.claimDue('worker-test', 1, 60_000)).requests; + await complete(claimed.id, claimed.lease_token, { online: true }); + await expect(enqueue({ + agentUrl: first.request.agent_url, + requestedByUserId: 'user-other', + requestedByAuthWorkosUserId: 'user-other', + })).rejects.toBeInstanceOf(ComplianceRefreshRateLimitError); + }); + it('claims once across workers and preserves session identity on lease recovery', async () => { const created = await enqueue(); const [workerA, workerB] = await Promise.all([ @@ -119,9 +167,9 @@ describe('durable compliance refresh request queue', () => { const [reclaimed] = (await db.claimDue('worker-c', 1, 60_000)).requests; expect(reclaimed.was_reclaimed).toBe(true); expect(reclaimed.test_session_id).toBe(firstClaim.test_session_id); - await expect(db.markSucceeded(created.request.id, firstClaim.lease_token, { online: true })) + await expect(complete(created.request.id, firstClaim.lease_token, { online: true })) .resolves.toBe(false); - await expect(db.markSucceeded(created.request.id, reclaimed.lease_token, { online: true })) + await expect(complete(created.request.id, reclaimed.lease_token, { online: true })) .resolves.toBe(true); }); @@ -170,4 +218,127 @@ describe('durable compliance refresh request queue', () => { scope: 'requester', }); }); + + it.each(['queued', 'succeeded'] as const)('rejects absent or malformed fingerprints before %s coalescing', async status => { + const first = await enqueue(); + if (status === 'succeeded') { + const [claim] = (await db.claimDue('worker', 1, 60_000)).requests; + await complete(claim.id, claim.lease_token, { online: true }); + } + for (const fingerprint of [undefined, null, '__unproven__', 'user-other:1', 'user-refresh-test:0', + 'user-refresh-test:01', 'user-refresh-test:9223372036854775808', 'user-refresh-test:1,user-other:2']) { + await expect(enqueue({ agentUrl: first.request.agent_url, authorizationFingerprint: fingerprint as string })) + .rejects.toBeInstanceOf(ComplianceRefreshProvenanceError); + } + await expect(enqueue({ agentUrl: first.request.agent_url, requesterType: 'static_admin', + requestedByUserId: null, requestedByAuthWorkosUserId: null })) + .rejects.toBeInstanceOf(ComplianceRefreshProvenanceError); + }); + + it('rejects missing local credential authority as unavailable', async () => { + await pool.query("DELETE FROM users WHERE workos_user_id = 'user-refresh-test'"); + await expect(enqueue()).rejects.toMatchObject({ code: 'authorization_unavailable' }); + }); + + it.each(['queued', 'running', 'succeeded', 'failed'] as const)('rejects stale epoch admission and replaces a revoked %s context', async status => { + const first = await enqueue(); + if (status !== 'queued') { + const [claim] = (await db.claimDue('worker', 1, 60_000)).requests; + if (status === 'succeeded') await complete(claim.id, claim.lease_token, { online: true }); + if (status === 'failed') await db.markFailed(claim.id, claim.lease_token, 'authorization_revoked', 'Revoked'); + } + await pool.query("INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"); + await expect(enqueue({ agentUrl: first.request.agent_url })).rejects.toMatchObject({ code: 'authorization_revoked' }); + const fresh = await enqueue({ agentUrl: first.request.agent_url, authorizationFingerprint: 'user-refresh-test:1', requesterLimit: 1 }); + expect(fresh.coalesced).toBe(false); + expect(fresh.request.id).not.toBe(first.request.id); + expect(fresh.request.authorization_fingerprint).toBe('user-refresh-test:1'); + if (status === 'queued' || status === 'running') { + expect(await db.getById(first.request.id)).toMatchObject({ status: 'failed', last_error_code: 'authorization_revoked' }); + } + }); + + it('retains a revoked active request until its execution fence is released', async () => { + const first = await enqueue(); + const fence = await db.acquireExecutionFence(first.request.id, first.request.agent_url); + await pool.query("INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"); + const replacement = { agentUrl: first.request.agent_url, authorizationFingerprint: 'user-refresh-test:1' }; + try { + await expect(enqueue(replacement)).rejects.toBeInstanceOf(ComplianceRefreshInProgressError); + expect((await db.getById(first.request.id))?.status).toBe('queued'); + } finally { + await fence?.release(); + } + expect((await enqueue(replacement)).coalesced).toBe(false); + }); + + it('reads an epoch committed while admission waited for its exact user lock', async () => { + const blocker = await pool.connect(); + await blocker.query('BEGIN'); + await blocker.query("SELECT workos_user_id FROM users WHERE workos_user_id = 'user-refresh-test' FOR UPDATE"); + await blocker.query("INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"); + const admission = enqueue().then(value => ({ value }), error => ({ error })); + try { + await vi.waitFor(async () => { + const waiting = await pool.query(`SELECT COUNT(*)::integer AS count FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE 'SELECT workos_user_id FROM users WHERE workos_user_id = $1 FOR UPDATE%'`); + expect(waiting.rows[0].count).toBeGreaterThan(0); + }); + await blocker.query('COMMIT'); + expect(await admission).toMatchObject({ error: { code: 'authorization_revoked' } }); + } finally { + await blocker.query('ROLLBACK'); + blocker.release(); + await admission; + } + }); + + it.each([false, true])('holds the %s existing-epoch fence against a concurrent mutation', async existing => { + if (existing) await pool.query("INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"); + const locked = await pool.connect(); + const writer = await pool.connect(); + try { + await locked.query('BEGIN'); + await assertComplianceRefreshAuthorizationFingerprint(locked, 'user-refresh-test', existing ? 'user-refresh-test:1' : ''); + await writer.query('BEGIN'); + await writer.query("SET LOCAL lock_timeout = '50ms'"); + const mutation = existing + ? "UPDATE authorization_epochs SET epoch = 2 WHERE workos_user_id = 'user-refresh-test'" + : "INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"; + await expect(writer.query(mutation)).rejects.toMatchObject({ code: '55P03' }); + await writer.query('ROLLBACK'); + await locked.query('COMMIT'); + await writer.query(mutation); + } finally { + await locked.query('ROLLBACK'); + await writer.query('ROLLBACK'); + locked.release(); + writer.release(); + } + }); + + it('rolls back completion writes when the same-transaction authorization callback rejects', async () => { + const first = await enqueue(); + const [claim] = (await db.claimDue('worker', 1, 60_000)).requests; + await expect(db.markSucceeded(first.request.id, claim.lease_token, { online: true }, async client => { + await client.query("UPDATE users SET email = 'must-rollback@example.test' WHERE workos_user_id = 'user-refresh-test'"); + throw Object.assign(new Error('Revoked'), { code: 'authorization_revoked' }); + })).rejects.toMatchObject({ code: 'authorization_revoked' }); + expect((await pool.query("SELECT email FROM users WHERE workos_user_id = 'user-refresh-test'")).rows[0].email) + .toBe('refresh@example.test'); + expect(await db.getById(first.request.id)).toMatchObject({ status: 'running', result_json: null }); + await expect(db.markSucceeded(first.request.id, claim.lease_token, {}, undefined as never)) + .rejects.toMatchObject({ code: 'authorization_unavailable' }); + }); + + it('refuses completion when its captured authorization epoch has changed', async () => { + const first = await enqueue(); + const [claim] = (await db.claimDue('worker', 1, 60_000)).requests; + await pool.query("INSERT INTO authorization_epochs (workos_user_id, epoch) VALUES ('user-refresh-test', 1)"); + await expect(complete(first.request.id, claim.lease_token, { online: true })) + .rejects.toMatchObject({ code: 'authorization_revoked' }); + expect(await db.getById(first.request.id)).toMatchObject({ status: 'running', result_json: null }); + }); + }); diff --git a/server/tests/integration/registry-api-agent-refresh.test.ts b/server/tests/integration/registry-api-agent-refresh.test.ts index d3bf1c11680..a410df4ef39 100644 --- a/server/tests/integration/registry-api-agent-refresh.test.ts +++ b/server/tests/integration/registry-api-agent-refresh.test.ts @@ -1,9 +1,9 @@ /** * Integration tests for POST /api/registry/agents/:encodedUrl/refresh. * - * Human refresh requests are fenced until durable credential provenance is - * supported. Static-admin refreshes retain the durable probe/compliance lifecycle; - * legacy human queue rows must fail before recovering or publishing evidence. + * Human refresh requests persist and recheck exact credential provenance. + * Static-admin and legacy unproven queue rows must fail before probing, + * resolving secrets, recovering runs, or publishing evidence. * * Run locally: * DATABASE_URL=postgresql://adcp:localdev@localhost:53198/adcp_test \ @@ -13,19 +13,21 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; import { randomUUID } from 'node:crypto'; -import { AAOAdminLookupUnavailableError } from '../../src/addie/admin-status-lookup.js'; import type { Pool } from 'pg'; import { HTTPServer } from '../../src/http.js'; import { initializeDatabase, closeDatabase } from '../../src/db/client.js'; import { runMigrations } from '../../src/db/migrate.js'; import { AAO_UA_COMPLIANCE } from '../../src/config/user-agents.js'; import { HOSTED_FULL_COMPLIANCE_TIMEOUT_MS } from '../../src/services/hosted-compliance-version.js'; -import { ComplianceRefreshRequestsDatabase } from '../../src/db/compliance-refresh-requests-db.js'; +import { + ComplianceRefreshRequestsDatabase, + type ComplianceRefreshRequestStatus, +} from '../../src/db/compliance-refresh-requests-db.js'; import { ComplianceDatabase } from '../../src/db/compliance-db.js'; -import { AgentContextDatabase } from '../../src/db/agent-context-db.js'; -import type { ComplianceRefreshQueue } from '../../src/services/compliance-refresh-queue.js'; import { complianceRunProvenance } from '../../src/compliance/run-provenance.js'; import type { ComplianceResult, ComplyOptions } from '@adcp/sdk/testing'; +import { bumpAuthorizationEpochs } from '../../src/db/authorization-epoch-db.js'; +import { isRefreshAdmin } from '../../src/services/compliance-refresh-authorization.js'; vi.hoisted(() => { process.env.WORKOS_API_KEY ??= 'sk_test_registry_refresh'; @@ -45,17 +47,7 @@ const SECOND_ORG_ID = `org_test_refresh_second_${RUN_SUFFIX}`; // expects it to succeed. const ownedAgentUrl = (slug: string) => `https://refresh-${slug}-${RUN_SUFFIX}.example.com/mcp`; const OTHER_AGENT_URL = `https://other-agent-${RUN_SUFFIX}.example.com/mcp`; -const LEGACY_USER_CASES = (['manual', 'owner_test'] as const).flatMap(triggeredBy => - (['queued', 'running'] as const).flatMap(status => - [false, true].map(checkpointed => ({ - triggeredBy, status, checkpointed, - slug: `legacy-${triggeredBy}-${status}-${checkpointed ? 'checkpoint' : 'fresh'}`, - })), - ), -); const ALL_OWNED_URLS = [ - ...LEGACY_USER_CASES.map(({ slug }) => ownedAgentUrl(slug)), - ownedAgentUrl('legacy-static-owner'), ownedAgentUrl('owner'), ownedAgentUrl('admin'), ownedAgentUrl('linked-owner'), @@ -80,6 +72,20 @@ const ALL_OWNED_URLS = [ ownedAgentUrl('legacy-timeout'), ownedAgentUrl('targeted-complete'), ownedAgentUrl('targeted-timed_out'), + ownedAgentUrl('linked-admin'), + ownedAgentUrl('deleted-credential'), + ownedAgentUrl('mismatched-credential'), + ownedAgentUrl('break-glass-email-changed'), + ownedAgentUrl('break-glass-email-current'), + ownedAgentUrl('authorization-outage'), + ownedAgentUrl('authorization-exhausted'), + ownedAgentUrl('legacy-provenance'), + ownedAgentUrl('admission-epoch-race'), + ownedAgentUrl('comply-epoch-revoked'), + ownedAgentUrl('comply-membership-revoked'), + ownedAgentUrl('polling-credential-deleted'), + ownedAgentUrl('polling-credential-mismatched'), + ownedAgentUrl('polling-credential-outage'), ]; // Toggle which user the auth middleware stamps onto the request. Tests @@ -87,15 +93,16 @@ const ALL_OWNED_URLS = [ let currentUserId: string | null = OWNER_USER_ID; let currentAuthWorkosUserId: string | undefined; let currentIsAdmin: boolean | undefined; -let currentRequestUser: { id: string; authWorkosUserId?: string; email: string; isAdmin?: boolean } | undefined; +let currentUserEmail: string | undefined; vi.mock('../../src/middleware/auth.js', async () => { const actual = await vi.importActual>('../../src/middleware/auth.js'); const stampUser = (req: { user?: unknown; isStaticAdminApiKey?: boolean }) => { if (currentUserId === null) return; - currentRequestUser = { + const currentRequestUser = { id: currentUserId, authWorkosUserId: currentAuthWorkosUserId, - email: `${currentAuthWorkosUserId ?? currentUserId}@test.com`, isAdmin: currentIsAdmin, + email: currentUserEmail ?? `${currentAuthWorkosUserId ?? currentUserId}@test.com`, + isAdmin: currentIsAdmin, }; req.user = currentRequestUser; if (currentUserId === STATIC_ADMIN_USER_ID) { @@ -145,6 +152,19 @@ vi.mock('../../src/addie/admin-status-lookup.js', async (importOriginal) => ({ isAuthenticatedUserAAOAdmin: (principal: { id: string; authWorkosUserId?: string }) => isAdminMock(principal), })); +const { getWorkosUserMock } = vi.hoisted(() => ({ + getWorkosUserMock: vi.fn(), +})); +vi.mock('../../src/auth/workos-client.js', async () => { + const actual = await vi.importActual('../../src/auth/workos-client.js'); + return { + ...actual, + getAuthorizationEnforcementWorkos: () => ({ + userManagement: { getUser: getWorkosUserMock }, + }), + }; +}); + // Stub the actual probe — the test doesn't need real outbound capability // discovery, only that the route plumbs the call through correctly. The // type-promotion / snapshot-write logic is exercised separately by the @@ -237,6 +257,22 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); await runMigrations(); + for (const userId of [OWNER_USER_ID, OTHER_USER_ID, ADMIN_USER_ID]) { + await pool.query( + 'INSERT INTO users (workos_user_id, email) VALUES ($1, $2)', + [userId, `${userId}@test.com`], + ); + } + await pool.query( + `INSERT INTO working_groups (name, slug, status) + VALUES ('AAO administrators', 'aao-admin', 'active') ON CONFLICT (slug) DO NOTHING`, + ); + await pool.query( + `INSERT INTO working_group_memberships (working_group_id, workos_user_id, status) + SELECT id, $1, 'active' FROM working_groups WHERE slug = 'aao-admin'`, + [ADMIN_USER_ID], + ); + await pool.query( `INSERT INTO organizations ( workos_organization_id, name, membership_tier, subscription_status, created_at, updated_at @@ -319,17 +355,25 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { await pool.query('DELETE FROM member_profiles WHERE workos_organization_id = ANY($1)', [[TEST_ORG_ID, SECOND_ORG_ID]]); await pool.query('DELETE FROM organization_memberships WHERE workos_organization_id = ANY($1)', [[TEST_ORG_ID, SECOND_ORG_ID]]); await pool.query('DELETE FROM organizations WHERE workos_organization_id = ANY($1)', [[TEST_ORG_ID, SECOND_ORG_ID]]); + await pool.query('DELETE FROM working_group_memberships WHERE workos_user_id = ANY($1)', [[OWNER_USER_ID, OTHER_USER_ID, ADMIN_USER_ID]]); + await pool.query('DELETE FROM users WHERE workos_user_id = ANY($1)', [[OWNER_USER_ID, OTHER_USER_ID, ADMIN_USER_ID]]); await server?.stop(); await closeDatabase(); }, 120_000); - beforeEach(() => { + beforeEach(async () => { currentUserId = OWNER_USER_ID; currentAuthWorkosUserId = undefined; currentIsAdmin = undefined; - currentRequestUser = undefined; isAdminMock.mockReset(); isAdminMock.mockImplementation(async (principal: { id: string; authWorkosUserId?: string }) => (principal.authWorkosUserId ?? principal.id) === ADMIN_USER_ID); + currentUserEmail = undefined; + currentIsAdmin = false; + // Queue authorization must read exact persisted membership rather than + // trusting presentation-layer isAdmin flags or canonical identities. + await pool.query('DELETE FROM authorization_epochs WHERE workos_user_id = ANY($1)', [[OWNER_USER_ID, OTHER_USER_ID, ADMIN_USER_ID]]); + getWorkosUserMock.mockReset(); + getWorkosUserMock.mockImplementation(async (id: string) => ({ id, email: `${id}@test.com` })); refreshSingleAgentMock.mockReset(); refreshSingleAgentMock.mockResolvedValue({ online: true, @@ -356,144 +400,42 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); const url = (agentUrl: string) => `/api/registry/agents/${encodeURIComponent(agentUrl)}/refresh`; - - const stopRefreshWorker = async () => { - const queue = (server as unknown as { complianceRefreshQueue: ComplianceRefreshQueue }).complianceRefreshQueue; - queue.stop(); - await vi.waitFor(() => { - expect((queue as unknown as { processing: boolean }).processing).toBe(false); - }); - return queue; - }; - - it.each(LEGACY_USER_CASES)( - 'terminally rejects legacy user $triggeredBy $status checkpointed=$checkpointed before any domain work', - async ({ triggeredBy, status, checkpointed, slug }) => { - const queue = await stopRefreshWorker(); - const operationId = randomUUID(); - const agentUrl = ownedAgentUrl(slug); - const runId = randomUUID(); - const canonicalUserId = triggeredBy === 'manual' ? ADMIN_USER_ID : OWNER_USER_ID; - const probe = checkpointed ? { - online: true, tools_count: 4, response_time_ms: 120, inferred_type: 'governance', - type_promoted: false, oauth_required: false, checked_at: new Date().toISOString(), - } : null; - const context = await new AgentContextDatabase().create({ - organization_id: TEST_ORG_ID, agent_url: agentUrl, created_by: OWNER_USER_ID, - }); - await new AgentContextDatabase().saveAuthToken(context.id, 'legacy-test-bearer-do-not-use-in-prod', 'bearer'); - await pool.query( - `INSERT INTO agent_compliance_refresh_requests - (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, - test_session_id, status, attempts, max_attempts, available_at, - lease_owner, lease_token, lease_expires_at, probe_result_json, auth_available) - VALUES ($1, $2, $3, 'user', $4, $5, $6, $7, $8, 5, NOW(), $9, $10, $11, $12::jsonb, $13)`, - [operationId, agentUrl, triggeredBy === 'owner_test' ? TEST_ORG_ID : null, - canonicalUserId, triggeredBy, `legacy-refresh-${operationId}`, status, - status === 'running' ? 1 : 0, status === 'running' ? 'expired-worker' : null, - status === 'running' ? randomUUID() : null, - status === 'running' ? new Date(Date.now() - 60_000) : null, - probe ? JSON.stringify(probe) : null, checkpointed ? true : null], - ); - if (checkpointed) { - await pool.query( - `INSERT INTO agent_compliance_runs - (id, agent_url, lifecycle_stage, overall_status, triggered_by, dry_run, refresh_operation_id) - VALUES ($1, $2, 'production', 'passing', $3, FALSE, $4)`, - [runId, agentUrl, triggeredBy, operationId], - ); - } - const domainState = () => pool.query( - `SELECT * FROM ( - SELECT 'runs' AS source, to_jsonb(t) AS value FROM agent_compliance_runs t WHERE agent_url = $1 - UNION ALL SELECT 'status', to_jsonb(t) FROM agent_compliance_status t WHERE agent_url = $1 - UNION ALL SELECT 'storyboards', to_jsonb(t) FROM agent_storyboard_status t WHERE agent_url = $1 - UNION ALL SELECT 'diagnostics', to_jsonb(t) FROM agent_compliance_step_diagnostics t WHERE agent_url = $1 - UNION ALL SELECT 'badges', to_jsonb(t) FROM agent_verification_badges t WHERE agent_url = $1 - UNION ALL SELECT 'health', to_jsonb(t) FROM agent_health_snapshot t WHERE agent_url = $1 - UNION ALL SELECT 'capabilities', to_jsonb(t) FROM agent_capabilities_snapshot t WHERE agent_url = $1 - UNION ALL SELECT 'credentials', to_jsonb(t) FROM agent_contexts t WHERE agent_url = $1) AS domain_state - ORDER BY source, value::text`, - [agentUrl], - ); - const before = await domainState(); - const forbiddenCalls = [ - vi.spyOn(ComplianceDatabase.prototype, 'getRunForRefreshOperation'), - vi.spyOn(ComplianceDatabase.prototype, 'resolveOwnerAuth'), - vi.spyOn(ComplianceDatabase.prototype, 'recordComplianceRun'), - vi.spyOn(ComplianceDatabase.prototype, 'upsertBadge'), - vi.spyOn(ComplianceDatabase.prototype, 'revokeBadge'), - vi.spyOn(ComplianceDatabase.prototype, 'revokeAllBadges'), - vi.spyOn(AgentContextDatabase.prototype, 'getByOrgAndUrl'), - vi.spyOn(AgentContextDatabase.prototype, 'getAuthInfoByOrgAndUrl'), - vi.spyOn(AgentContextDatabase.prototype, 'getOAuthTokensByOrgAndUrl'), - vi.spyOn(AgentContextDatabase.prototype, 'getOAuthClientCredentialsByOrgAndUrl'), - vi.spyOn(AgentContextDatabase.prototype, 'findOwnerOrgWithSavedAuth'), - vi.spyOn(ComplianceRefreshRequestsDatabase.prototype, 'recordProbeResult'), - vi.spyOn(ComplianceRefreshRequestsDatabase.prototype, 'markSucceeded'), - vi.spyOn(ComplianceRefreshRequestsDatabase.prototype, 'requeueAfterFailure'), - ]; - try { - expect(await queue.processQueue()).toEqual({ claimed: 1, succeeded: 0, failed: 1, lostLease: 0 }); - const failed = await pool.query('SELECT * FROM agent_compliance_refresh_requests WHERE id = $1', [operationId]); - expect(failed.rows[0]).toMatchObject({ - status: 'failed', attempts: status === 'running' ? 2 : 1, - last_error_code: 'authorization_provenance_missing', - last_error: 'Refresh requester authorization provenance is unavailable', - lease_owner: null, lease_token: null, lease_expires_at: null, - result_json: null, probe_result_json: probe, - }); - expect(failed.rows[0].completed_at).toBeInstanceOf(Date); - expect(await queue.processQueue()).toEqual({ claimed: 0, succeeded: 0, failed: 0, lostLease: 0 }); - expect((await pool.query('SELECT attempts FROM agent_compliance_refresh_requests WHERE id = $1', [operationId])).rows[0].attempts) - .toBe(status === 'running' ? 2 : 1); - for (const spy of forbiddenCalls) expect(spy).not.toHaveBeenCalled(); - expect(isAdminMock).not.toHaveBeenCalled(); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await domainState()).rows).toEqual(before.rows); - - currentUserId = STATIC_ADMIN_USER_ID; - const response = await request(app).get(`${url(agentUrl)}es/${operationId}`).send(); - expect(response.status).toBe(200); - expect(response.body).toMatchObject({ - status: 'failed', - error: { - code: 'authorization_provenance_missing', - message: 'Refresh requester authorization provenance is unavailable', - }, - }); - } finally { - for (const spy of forbiddenCalls) spy.mockRestore(); - await pool.query('DELETE FROM agent_compliance_runs WHERE id = $1', [runId]); - await pool.query('DELETE FROM agent_compliance_refresh_requests WHERE id = $1', [operationId]); - await pool.query('DELETE FROM agent_contexts WHERE id = $1', [context.id]); - queue.start(); - } - }, - ); - - it('preserves rejection of a legacy static-admin row carrying an owner context without a requester', async () => { - const queue = await stopRefreshWorker(); - const operationId = randomUUID(); - const agentUrl = ownedAgentUrl('legacy-static-owner'); + const refreshDb = new ComplianceRefreshRequestsDatabase(); + const withLinkedCredential = async (canonicalId: string, credentialId: string, test: () => Promise) => { + const original = await pool.query<{ identity_id: string }>( + 'SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1', + [credentialId], + ); + await pool.query( + `UPDATE identity_workos_users SET identity_id = ( + SELECT identity_id FROM identity_workos_users WHERE workos_user_id = $1 + ), is_primary = FALSE WHERE workos_user_id = $2`, + [canonicalId, credentialId], + ); try { + await test(); + } finally { await pool.query( - `INSERT INTO agent_compliance_refresh_requests - (id, agent_url, owner_org_id, requester_type, requested_by_user_id, triggered_by, test_session_id) - VALUES ($1, $2, $3, 'static_admin', NULL, 'owner_test', $4)`, - [operationId, agentUrl, TEST_ORG_ID, `legacy-refresh-${operationId}`], + 'UPDATE identity_workos_users SET identity_id = $1, is_primary = TRUE WHERE workos_user_id = $2', + [original.rows[0].identity_id, credentialId], ); - expect(await queue.processQueue()).toEqual({ claimed: 1, succeeded: 0, failed: 1, lostLease: 0 }); - const failed = await pool.query('SELECT status, last_error_code FROM agent_compliance_refresh_requests WHERE id = $1', [operationId]); - expect(failed.rows[0]).toEqual({ status: 'failed', last_error_code: 'authorization_revoked' }); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - } finally { - await pool.query('DELETE FROM agent_compliance_refresh_requests WHERE id = $1', [operationId]); - queue.start(); } - }); + }; + const waitForOperation = async ( + operationId: string, + status: ComplianceRefreshRequestStatus, + errorCode?: string, + ) => { + const deadline = Date.now() + 12_000; + while (Date.now() < deadline) { + const operation = await refreshDb.getById(operationId); + if (operation?.status === status && (!errorCode || operation.last_error_code === errorCode)) { + return operation; + } + await new Promise(resolve => setTimeout(resolve, 25)); + } + throw new Error(`Refresh did not reach ${status}${errorCode ? ` (${errorCode})` : ''}`); + }; it.each(['complete', 'timed_out'] as const)('reports targeted %s runs as audit-only with provenance and no diagnostics', async completeness => { const agentUrl = ownedAgentUrl(`targeted-${completeness}`); @@ -515,8 +457,8 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { .toBe(res.body.provenance.test_session_id); }); - it('static admin can refresh and gets the snapshot back', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + it('authenticated administrator can refresh and gets the snapshot back', async () => { + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('owner'); const res = await request(app).post(url(agentUrl)).send(); expect(res.status).toBe(200); @@ -584,7 +526,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); it('public compliance bounds notice output while retaining the raw private record', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('public-notices'); const refresh = await request(app).post(url(agentUrl)).send(); expect(refresh.status).toBe(200); @@ -655,158 +597,375 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { expect(stored.rows[0].notices_json[0]).toHaveProperty('experimental_context'); }); - it('fences administrator session refresh until durable credential provenance is supported', async () => { - currentUserId = ADMIN_USER_ID; - const agentUrl = ownedAgentUrl('admin'); - const res = await request(app).post(url(agentUrl)).send(); - expect(res.status).toBe(503); - expect(res.body.code).toBe('refresh_authorization_provenance_required'); - expect(res.headers['retry-after']).toBe('60'); - expect(res.headers['cache-control']).toBe('private, no-store'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - const queued = await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl]); - expect(queued.rows).toEqual([]); + it.each([ + ['owner', OWNER_USER_ID], + ['admin', ADMIN_USER_ID], + ])('does not inherit a linked canonical %s credential at admission', async (_role, canonicalId) => { + currentUserId = canonicalId; + currentAuthWorkosUserId = OTHER_USER_ID; + currentIsAdmin = true; + + await withLinkedCredential(canonicalId, OTHER_USER_ID, async () => { + const res = await request(app).post(url(ownedAgentUrl('linked-owner'))).send(); + + expect(res.status).toBe(403); + expect(getWorkosUserMock).toHaveBeenCalledWith(OTHER_USER_ID); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === OTHER_USER_ID)).toBe(true); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + }); }); it.each([ - { authenticated: ADMIN_USER_ID, canonical: OTHER_USER_ID, status: 503 }, - { authenticated: OTHER_USER_ID, canonical: ADMIN_USER_ID, status: 403 }, - ])('uses exact $authenticated admin authorization linked to $canonical before refresh admission', async ({ authenticated, canonical, status }) => { - currentUserId = canonical; - currentAuthWorkosUserId = authenticated; - currentIsAdmin = true; - const response = await request(app).post(url(OTHER_AGENT_URL)).send(); - expect(response.status).toBe(status); - if (status === 503) expect(response.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - const queued = await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [OTHER_AGENT_URL]); - expect(queued.rows).toEqual([]); - // The rate limiter can independently check its optional admin exemption; - // the route must additionally resolve its immutable captured principal. - const principals = isAdminMock.mock.calls.map(([principal]) => principal); - expect(principals.every(principal => (principal.authWorkosUserId ?? principal.id) === authenticated)).toBe(true); - const routePrincipal = principals.find(principal => Object.isFrozen(principal)); - expect(routePrincipal).toMatchObject({ id: authenticated }); + [ADMIN_USER_ID, OTHER_USER_ID, false], + [OTHER_USER_ID, ADMIN_USER_ID, true], + ])('checks raw administrator membership with canonical %s and authenticated %s', async (canonicalId, credentialId, authorized) => { + await withLinkedCredential(canonicalId, credentialId, async () => { + await expect(isRefreshAdmin({ id: canonicalId, authWorkosUserId: credentialId })) + .resolves.toBe(authorized); + expect(getWorkosUserMock).toHaveBeenCalledWith(credentialId); + }); }); - it('fences linked owner refresh instead of persisting ambiguous credential provenance', async () => { + it.each([ + ['owner', OWNER_USER_ID, 'owner_test'], + ['admin', ADMIN_USER_ID, 'manual'], + ])('retains the authenticated %s credential through admission, execution, and polling', async (role, authenticatedId, triggeredBy) => { + const agentUrl = ownedAgentUrl(`linked-${role}`); currentUserId = OTHER_USER_ID; - currentAuthWorkosUserId = OWNER_USER_ID; - const agentUrl = ownedAgentUrl('linked-owner'); - const response = await request(app).post(url(agentUrl)).send(); - expect(response.status).toBe(503); - expect(response.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); + currentAuthWorkosUserId = authenticatedId; + + await withLinkedCredential(OTHER_USER_ID, authenticatedId, async () => { + const accepted = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(accepted.status).toBe(202); + const operation = await waitForOperation(accepted.body.refresh_operation_id, 'succeeded'); + expect(operation).toMatchObject({ + requester_type: 'user', + requested_by_user_id: authenticatedId, + requested_by_auth_workos_user_id: authenticatedId, + authorization_fingerprint: '', + triggered_by: triggeredBy, + }); + expect(getWorkosUserMock).toHaveBeenCalledWith(authenticatedId); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === authenticatedId)).toBe(true); + expect(refreshSingleAgentMock).toHaveBeenCalledOnce(); + expect(complyMock).toHaveBeenCalledOnce(); + + for (const canonicalId of [OWNER_USER_ID, ADMIN_USER_ID]) { + currentUserId = canonicalId; + currentAuthWorkosUserId = OTHER_USER_ID; + currentIsAdmin = true; + const forbidden = await request(app).get(accepted.body.status_url).send(); + expect(forbidden.status).toBe(404); + } + + currentUserId = OTHER_USER_ID; + currentAuthWorkosUserId = authenticatedId; + currentIsAdmin = false; + const completed = await request(app).get(accepted.body.status_url).send(); + expect(completed.status).toBe(200); + expect(completed.body.status).toBe('succeeded'); + }); }); - it('does not inherit canonical organization ownership on refresh', async () => { - currentUserId = OWNER_USER_ID; - currentAuthWorkosUserId = OTHER_USER_ID; - const agentUrl = ownedAgentUrl('linked-owner'); - const response = await request(app).post(url(agentUrl)).send(); - expect(response.status).toBe(403); + it.each(['deleted', 'mismatched'])('rejects a %s authenticated WorkOS credential at execution before probing', async (state) => { + const agentUrl = ownedAgentUrl(`${state}-credential`); + currentUserId = OTHER_USER_ID; + currentAuthWorkosUserId = OWNER_USER_ID; + getWorkosUserMock.mockResolvedValueOnce({ id: OWNER_USER_ID, email: `${OWNER_USER_ID}@test.com` }); + if (state === 'deleted') { + getWorkosUserMock.mockRejectedValue(Object.assign(new Error('WorkOS user no longer exists'), { status: 404 })); + } else { + getWorkosUserMock.mockResolvedValue({ id: OTHER_USER_ID, email: `${OWNER_USER_ID}@test.com` }); + } + + const res = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + + expect(res.status).toBe(202); + const operation = await waitForOperation( + res.body.refresh_operation_id, + 'failed', + 'authorization_revoked', + ); + expect(getWorkosUserMock.mock.calls).toEqual([[OWNER_USER_ID], [OWNER_USER_ID]]); expect(refreshSingleAgentMock).not.toHaveBeenCalled(); expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); + expect(operation).toMatchObject({ + status: 'failed', attempts: 1, result_json: null, last_error_code: 'authorization_revoked', + }); + const status = await request(app).get(res.body.status_url).send(); + expect(status.body).toMatchObject({ + status: 'failed', attempts: 1, result: null, error: { code: 'authorization_revoked' }, + }); }); - it('fences a confirmed unlinked owner without misreporting a separate platform-admin outage', async () => { - isAdminMock.mockRejectedValue(new AAOAdminLookupUnavailableError()); - const agentUrl = ownedAgentUrl('owner-admin-outage'); - const response = await request(app).post(url(agentUrl)).send(); - expect(response.status).toBe(503); - expect(response.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); - expect(isAdminMock.mock.calls.every(([principal]) => (principal.authWorkosUserId ?? principal.id) === OWNER_USER_ID)).toBe(true); + it.each([ + ['deleted', 404, 403, 'authorization_revoked'], + ['mismatched', null, 403, 'authorization_revoked'], + ['outage', 503, 503, 'authorization_unavailable'], + ])('preserves %s credential semantics when an administrator polls a manual refresh', async (state, workosStatus, httpStatus, code) => { + const agentUrl = ownedAgentUrl(`polling-credential-${state}`); + currentUserId = OTHER_USER_ID; + currentAuthWorkosUserId = ADMIN_USER_ID; + const accepted = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(accepted.status).toBe(202); + await waitForOperation(accepted.body.refresh_operation_id, 'succeeded'); + getWorkosUserMock.mockClear(); + if (state === 'mismatched') { + getWorkosUserMock.mockResolvedValue({ id: OTHER_USER_ID, email: `${ADMIN_USER_ID}@test.com` }); + } else { + getWorkosUserMock.mockRejectedValue(Object.assign(new Error('Private WorkOS failure detail'), { status: workosStatus })); + } + + const status = await request(app).get(accepted.body.status_url).send(); + expect(status.status).toBe(httpStatus); + expect(status.body.code).toBe(code); + expect(status.headers['retry-after']).toBe(httpStatus === 503 ? '5' : undefined); + expect(JSON.stringify(status.body)).not.toContain('Private WorkOS failure detail'); + expect(getWorkosUserMock).toHaveBeenCalledExactlyOnceWith(ADMIN_USER_ID); }); - it('reports non-owner admin lookup unavailability without enqueuing or trusting isAdmin', async () => { + it.each([ + ['changed', 'ordinary@test.com', 403], + ['current', 'break-glass@test.com', 200], + ])('uses the %s WorkOS email for break-glass despite stale local and session data', async (state, workosEmail, expectedStatus) => { + const agentUrl = ownedAgentUrl(`break-glass-email-${state}`); + const sessionEmail = 'break-glass@test.com'; + const localEmail = state === 'changed' ? sessionEmail : 'stale-local@test.com'; currentUserId = ADMIN_USER_ID; currentAuthWorkosUserId = OTHER_USER_ID; + currentUserEmail = sessionEmail; currentIsAdmin = true; - isAdminMock.mockRejectedValue(new AAOAdminLookupUnavailableError()); - const response = await request(app).post(url(OTHER_AGENT_URL)).send(); - expect(response.status).toBe(503); - expect(response.body.error).toBe('admin_authorization_unavailable'); - expect(response.headers['retry-after']).toBe('5'); - expect(response.headers['cache-control']).toBe('no-store'); + vi.stubEnv('ADMIN_EMAILS', sessionEmail); + getWorkosUserMock.mockResolvedValue({ id: OTHER_USER_ID, email: workosEmail }); + await pool.query( + 'UPDATE users SET email = $2 WHERE workos_user_id = $1', + [OTHER_USER_ID, localEmail], + ); + try { + const res = await request(app).post(url(agentUrl)).send(); + + expect(res.status).toBe(expectedStatus); + expect(getWorkosUserMock).toHaveBeenCalledWith(OTHER_USER_ID); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === OTHER_USER_ID)).toBe(true); + if (expectedStatus === 403) { + expect(res.body.code).toBe('authorization_revoked'); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + } else { + expect(refreshSingleAgentMock).toHaveBeenCalledOnce(); + expect(complyMock).toHaveBeenCalledOnce(); + } + } finally { + await pool.query('UPDATE users SET email = $2 WHERE workos_user_id = $1', [OTHER_USER_ID, `${OTHER_USER_ID}@test.com`]); + vi.unstubAllEnvs(); + } + }); + + it('retries unavailable WorkOS authorization without executing until the exact credential is verified', async () => { + const agentUrl = ownedAgentUrl('authorization-outage'); + getWorkosUserMock + .mockResolvedValueOnce({ id: OWNER_USER_ID, email: `${OWNER_USER_ID}@test.com` }) + .mockRejectedValueOnce(Object.assign(new Error('WorkOS unavailable'), { status: 503 })); + + const accepted = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(accepted.status).toBe(202); + const retrying = await waitForOperation(accepted.body.refresh_operation_id, 'queued', 'authorization_unavailable'); + expect(retrying.attempts).toBe(1); expect(refreshSingleAgentMock).not.toHaveBeenCalled(); expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [OTHER_AGENT_URL])).rows).toEqual([]); + const pending = await request(app).get(accepted.body.status_url).send(); + expect(pending.status).toBe(200); + expect(pending.body).toMatchObject({ status: 'queued', attempts: 1, result: null }); + + await pool.query('UPDATE agent_compliance_refresh_requests SET available_at = NOW() WHERE id = $1', [retrying.id]); + const completed = await waitForOperation(retrying.id, 'succeeded'); + expect(completed.attempts).toBe(2); + expect(getWorkosUserMock.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === OWNER_USER_ID)).toBe(true); + expect(refreshSingleAgentMock).toHaveBeenCalledOnce(); + expect(complyMock).toHaveBeenCalledOnce(); }); - it.each([ - { authenticated: ADMIN_USER_ID, canonical: OTHER_USER_ID, allowed: true }, - { authenticated: OTHER_USER_ID, canonical: ADMIN_USER_ID, allowed: false }, - ])('authorizes refresh status with exact $authenticated and immutable lookup provenance', async ({ authenticated, canonical, allowed }) => { - const operationId = randomUUID(); + it('preserves authorization unavailable after retry exhaustion in polling and repeated POST responses', async () => { + const agentUrl = ownedAgentUrl('authorization-exhausted'); + getWorkosUserMock + .mockResolvedValueOnce({ id: OWNER_USER_ID, email: `${OWNER_USER_ID}@test.com` }) + .mockRejectedValue(Object.assign(new Error('Internal WorkOS outage detail'), { status: 503 })); + + const accepted = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(accepted.status).toBe(202); + const retrying = await waitForOperation( + accepted.body.refresh_operation_id, + 'queued', + 'authorization_unavailable', + ); await pool.query( - `INSERT INTO agent_compliance_refresh_requests - (id, agent_url, requester_type, requested_by_user_id, triggered_by, test_session_id, status, result_json, completed_at) - VALUES ($1, $2, 'user', $3, 'manual', $4, 'succeeded', '{"online":true}'::jsonb, NOW())`, - [operationId, OTHER_AGENT_URL, OWNER_USER_ID, `test-status-${operationId}`], + 'UPDATE agent_compliance_refresh_requests SET available_at = NOW() WHERE id = $1', + [retrying.id], ); - currentUserId = canonical; - currentAuthWorkosUserId = authenticated; - currentIsAdmin = !allowed; - isAdminMock.mockImplementationOnce(async (principal) => { - await Promise.resolve(); - currentRequestUser!.id = authenticated; - currentRequestUser!.authWorkosUserId = canonical; - return principal.id === ADMIN_USER_ID; + await waitForOperation(retrying.id, 'failed', 'authorization_unavailable'); + + const status = await request(app).get(accepted.body.status_url).send(); + expect(status.status).toBe(200); + expect(status.body).toMatchObject({ + status: 'failed', + attempts: 2, + result: null, + error: { code: 'authorization_unavailable' }, }); + expect(JSON.stringify(status.body)).not.toContain('Internal WorkOS outage detail'); + + const repeated = await request(app).post(url(agentUrl)).send(); + expect(repeated.status).toBe(503); + expect(repeated.body.code).toBe('authorization_unavailable'); + expect(getWorkosUserMock.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === OWNER_USER_ID)).toBe(true); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + }); + + it('preserves public missing-provenance rejection and permits a fresh valid resubmission', async () => { + const agentUrl = ownedAgentUrl('legacy-provenance'); + const operationId = randomUUID(); + const client = await pool.connect(); try { - const response = await request(app).get(`${url(OTHER_AGENT_URL)}es/${operationId}`).send(); - expect(response.status).toBe(allowed ? 200 : 404); - if (allowed) expect(response.body.result).toEqual({ online: true }); - const principal = isAdminMock.mock.calls[0][0]; - expect(principal.id).toBe(authenticated); - expect(Object.isFrozen(principal)).toBe(true); + await client.query('BEGIN'); + await client.query( + `INSERT INTO agent_compliance_refresh_requests + (id, agent_url, owner_org_id, requester_type, requested_by_user_id, + requested_by_auth_workos_user_id, authorization_fingerprint, triggered_by, test_session_id) + VALUES ($1, $2, $3, 'user', $4, $4, '', 'owner_test', $5)`, + [operationId, agentUrl, TEST_ORG_ID, OWNER_USER_ID, `owner-refresh-${operationId}`], + ); + await client.query( + `UPDATE agent_compliance_refresh_requests + SET status = 'failed', last_error_code = 'authorization_provenance_missing', completed_at = NOW() + WHERE id = $1`, + [operationId], + ); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; } finally { - await pool.query('DELETE FROM agent_compliance_refresh_requests WHERE id = $1', [operationId]); + client.release(); } - }); - it('static admin API key can refresh and rerun compliance for an agent it does not own', async () => { - currentUserId = STATIC_ADMIN_USER_ID; - const agentUrl = ownedAgentUrl('static-admin'); + const rejected = await waitForOperation(operationId, 'failed', 'authorization_provenance_missing'); + // Actual pre-migration NULL conversion/rejection is covered by the + // migration fixture; post-migration rows cannot contain NULL provenance. + expect(rejected.requested_by_auth_workos_user_id).toBe(OWNER_USER_ID); + expect(getWorkosUserMock).not.toHaveBeenCalled(); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + const status = await request(app) + .get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/refreshes/${operationId}`) + .send(); + expect(status.status).toBe(200); + expect(status.body).toMatchObject({ + status: 'failed', + error: { code: 'authorization_provenance_missing' }, + }); - const res = await request(app).post(url(agentUrl)).send(); + const fresh = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(fresh.status).toBe(202); + expect(fresh.body.refresh_operation_id).not.toBe(operationId); + expect(fresh.body.coalesced).toBe(false); + const completed = await waitForOperation(fresh.body.refresh_operation_id, 'succeeded'); + expect(completed.requested_by_auth_workos_user_id).toBe(OWNER_USER_ID); + expect(getWorkosUserMock).toHaveBeenCalledWith(OWNER_USER_ID); + expect(getWorkosUserMock.mock.calls.every(([id]) => id === OWNER_USER_ID)).toBe(true); + expect(refreshSingleAgentMock).toHaveBeenCalledOnce(); + expect(complyMock).toHaveBeenCalledOnce(); + }); - expect(res.status).toBe(200); - expect(res.body.compliance).toMatchObject({ - ran: true, - overall_status: 'passing', - storyboards_passing: 1, - storyboards_total: 1, + it('rejects an epoch change during admission before enqueueing or resolving saved credentials', async () => { + const agentUrl = ownedAgentUrl('admission-epoch-race'); + const resolveOwnerAuth = vi.spyOn(ComplianceDatabase.prototype, 'resolveOwnerAuth'); + getWorkosUserMock.mockImplementationOnce(async (id: string) => { + await bumpAuthorizationEpochs(pool, [id]); + return { id, email: `${id}@test.com` }; }); - expect(refreshSingleAgentMock).toHaveBeenCalledWith(agentUrl, expect.any(Object)); - expect(complyMock).toHaveBeenCalledWith( - agentUrl, - expect.objectContaining({ - timeout_ms: HOSTED_FULL_COMPLIANCE_TIMEOUT_MS, - userAgent: AAO_UA_COMPLIANCE, - }), - ); + try { + const response = await request(app).post(url(agentUrl)).send(); + expect(response.status).toBe(403); + expect(response.body.code).toBe('authorization_revoked'); + expect(resolveOwnerAuth).not.toHaveBeenCalled(); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + const queued = await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl]); + expect(queued.rows).toEqual([]); + } finally { + resolveOwnerAuth.mockRestore(); + } + }); - const latestRun = await pool.query( - `SELECT triggered_by, triggered_org_id - FROM agent_compliance_runs - WHERE agent_url = $1 - ORDER BY tested_at DESC - LIMIT 1`, - [agentUrl], - ); - expect(latestRun.rows[0]).toMatchObject({ - triggered_by: 'manual', - triggered_org_id: null, + it.each(['epoch', 'membership'])('rejects %s revocation while comply is pending without canonical writes or success', async (revocation) => { + const agentUrl = ownedAgentUrl(`comply-${revocation}-revoked`); + let releaseCompliance!: (value: ReturnType) => void; + let complianceStarted!: () => void; + const started = new Promise(resolve => { complianceStarted = resolve; }); + complyMock.mockImplementationOnce(() => { + complianceStarted(); + return new Promise(resolve => { releaseCompliance = resolve; }); }); + const result = makeComplianceResult({ specialisms: ['sales-broadcast-tv'], storyboardId: 'sales_broadcast_tv' }); + try { + const accepted = await request(app).post(url(agentUrl)).set('Prefer', 'respond-async').send(); + expect(accepted.status).toBe(202); + await started; + if (revocation === 'epoch') { + await bumpAuthorizationEpochs(pool, [OWNER_USER_ID]); + } else { + await pool.query( + 'DELETE FROM organization_memberships WHERE workos_organization_id = $1 AND workos_user_id = $2', + [TEST_ORG_ID, OWNER_USER_ID], + ); + } + releaseCompliance(result); + const failed = await waitForOperation(accepted.body.refresh_operation_id, 'failed', 'authorization_revoked'); + expect(failed).toMatchObject({ attempts: 1, result_json: null }); + for (const table of ['agent_compliance_runs', 'agent_compliance_status', 'agent_storyboard_status', 'agent_verification_badges']) { + const stored = await pool.query(`SELECT agent_url FROM ${table} WHERE agent_url = $1`, [agentUrl]); + expect(stored.rows, table).toEqual([]); + } + currentUserId = ADMIN_USER_ID; + const status = await request(app).get(accepted.body.status_url).send(); + expect(status.status).toBe(200); + expect(status.body).toMatchObject({ + status: 'failed', attempts: 1, result: null, error: { code: 'authorization_revoked' }, + }); + expect(complyMock).toHaveBeenCalledOnce(); + } finally { + releaseCompliance?.(result); + if (revocation === 'membership') { + await pool.query( + `INSERT INTO organization_memberships (workos_organization_id, workos_user_id, email, role) + VALUES ($1, $2, $3, 'admin') ON CONFLICT (workos_organization_id, workos_user_id) DO NOTHING`, + [TEST_ORG_ID, OWNER_USER_ID, `${OWNER_USER_ID}@test.com`], + ); + } + } + }); + + it.each(['rotated-key', ''])('rejects stale static administrator authorization after the key becomes %s', async (currentKey) => { + currentUserId = STATIC_ADMIN_USER_ID; + const agentUrl = ownedAgentUrl('static-admin'); + vi.stubEnv('ADMIN_API_KEY', currentKey); + try { + // Middleware deliberately stamps the stale static principal. Queue + // admission must still require revocable per-credential provenance. + const res = await request(app).post(url(agentUrl)).send(); + expect(res.status).toBe(403); + expect(getWorkosUserMock).not.toHaveBeenCalled(); + expect(refreshSingleAgentMock).not.toHaveBeenCalled(); + expect(complyMock).not.toHaveBeenCalled(); + const queued = await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl]); + expect(queued.rows).toEqual([]); + } finally { + vi.unstubAllEnvs(); + } }); it('non-owner non-admin gets 403', async () => { @@ -836,7 +995,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); it('returns 502 when the probe throws', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; refreshSingleAgentMock.mockRejectedValue(new Error('Probe timeout')); const res = await request(app).post(url(ownedAgentUrl('probe-fail'))).send(); expect(res.status).toBe(502); @@ -847,7 +1006,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); it('returns 409 when monitoring is paused', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; refreshSingleAgentMock.mockRejectedValue(new Error('Monitoring paused for this agent')); const res = await request(app).post(url(ownedAgentUrl('paused'))).send(); expect(res.status).toBe(409); @@ -855,7 +1014,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }); it('returns an immediate durable handle and polls a long-running refresh to completion', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('async-refresh'); let resolveCompliance!: (value: ReturnType) => void; const deferredCompliance = new Promise>((resolve) => { @@ -913,7 +1072,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }, 20_000); it('falls back to 202 at the legacy deadline without Prefer, then completes by polling', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('legacy-timeout'); let resolveCompliance!: (value: ReturnType) => void; const deferredCompliance = new Promise>((resolve) => { @@ -951,7 +1110,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }, 20_000); it.each(['complete', 'timed_out'] as const)('recovers a persisted %s run without executing a second suite or publishing partial evidence', async completeness => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl(completeness === 'complete' ? 'refresh-recovery' : 'refresh-recovery-timed_out'); complyMock.mockResolvedValueOnce({ ...makeComplianceResult(), completeness }); const revokeBadges = vi.spyOn(ComplianceDatabase.prototype, 'revokeBadge'); @@ -1037,6 +1196,8 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { expect(materialized.rowCount).toBe(0); const history = await request(app).get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/compliance/history`); expect(history.body.runs).toEqual([]); + currentUserId = OWNER_USER_ID; + currentAuthWorkosUserId = undefined; const diagnostics = await request(app).get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/compliance/diagnostics`); expect(diagnostics.body).toMatchObject({ completeness: 'timed_out', is_authoritative: false, diagnostics_visibility: 'owner_or_operator', provenance: { sdk_version: '14.0.0-rc.35', agent_build_version: null } }); @@ -1050,7 +1211,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }, 30_000); it('retries failed badge persistence from the saved run without executing a second suite', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('badge-retry'); complyMock.mockResolvedValueOnce(makeComplianceResult({ specialisms: ['sales-broadcast-tv'], @@ -1090,7 +1251,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }, 40_000); it('surfaces an allowlisted badge failure after retry exhaustion', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('badge-retry-exhausted'); complyMock.mockResolvedValueOnce(makeComplianceResult({ specialisms: ['sales-broadcast-tv'], @@ -1133,7 +1294,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { }, 40_000); it('recovers the same operation handle when a completed response is retried', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('rate-limit'); const first = await request(app).post(url(agentUrl)).send(); expect(first.status).toBe(200); @@ -1154,7 +1315,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { // even though evaluate_agent_quality (which resolves saved auth) worked // fine. The route now resolves owner-org auth and threads it to the // crawler so the probe sees the same credentials. - it('fences owner refresh before using an organization-saved bearer token', async () => { + it('threads the organization-saved bearer token through the authorized refresh', async () => { const agentUrl = ownedAgentUrl('saved-bearer'); const { AgentContextDatabase } = await import('../../src/db/agent-context-db.js'); const db = new AgentContextDatabase(); @@ -1168,17 +1329,25 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { try { const res = await request(app).post(url(agentUrl)).send(); - expect(res.status).toBe(503); - expect(res.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); + expect(res.status).toBe(200); + expect(refreshSingleAgentMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ + auth: { type: 'bearer', token: FAKE_BEARER }, + ownerOrgId: TEST_ORG_ID, + authorization: expect.any(Object), + }), + ); + expect(complyMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ auth: { type: 'bearer', token: FAKE_BEARER } }), + ); } finally { await pool.query('DELETE FROM agent_contexts WHERE id = $1', [context.id]); } }); - it('fences a canonicalized owner URL before resolving saved auth or probing', async () => { + it('canonicalizes the requested URL before authorized saved-auth lookup and probing', async () => { const agentUrl = ownedAgentUrl('canonical-saved-bearer'); const requestedUrl = agentUrl .replace('https://', 'HTTPS://') @@ -1195,11 +1364,19 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { try { const res = await request(app).post(url(requestedUrl)).send(); - expect(res.status).toBe(503); - expect(res.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); + expect(res.status).toBe(200); + expect(refreshSingleAgentMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ + auth: { type: 'bearer', token: FAKE_BEARER }, + ownerOrgId: TEST_ORG_ID, + authorization: expect.any(Object), + }), + ); + expect(complyMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ auth: { type: 'bearer', token: FAKE_BEARER } }), + ); } finally { await pool.query('DELETE FROM agent_contexts WHERE id = $1', [context.id]); } @@ -1240,7 +1417,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { } }); - it('fences selected-organization owner refresh before using saved credentials', async () => { + it('uses selected-organization credentials after exact authorization checks', async () => { const agentUrl = ownedAgentUrl('selected-org-refresh'); const { AgentContextDatabase } = await import('../../src/db/agent-context-db.js'); const db = new AgentContextDatabase(); @@ -1257,11 +1434,19 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { .post(url(agentUrl)) .send({ organization_id: SECOND_ORG_ID }); - expect(res.status).toBe(503); - expect(res.body.code).toBe('refresh_authorization_provenance_required'); - expect(refreshSingleAgentMock).not.toHaveBeenCalled(); - expect(complyMock).not.toHaveBeenCalled(); - expect((await pool.query('SELECT id FROM agent_compliance_refresh_requests WHERE agent_url = $1', [agentUrl])).rows).toEqual([]); + expect(res.status).toBe(200); + expect(refreshSingleAgentMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ + auth: { type: 'bearer', token }, + ownerOrgId: SECOND_ORG_ID, + authorization: expect.any(Object), + }), + ); + expect(complyMock).toHaveBeenCalledWith( + agentUrl, + expect.objectContaining({ auth: { type: 'bearer', token } }), + ); } finally { await pool.query('DELETE FROM agent_contexts WHERE id = $1', [context.id]); } @@ -1297,8 +1482,8 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { expect(stored.rows).toEqual([{ organization_id: SECOND_ORG_ID }]); }); - it('fans out badge issuance for a static-admin refresh with a passing specialism', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + it('fans out badge issuance for an authenticated administrator refresh with a passing specialism', async () => { + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('badge-fanout'); complyMock.mockResolvedValueOnce(makeComplianceResult({ specialisms: ['sales-broadcast-tv'], @@ -1336,7 +1521,7 @@ describe('POST /api/registry/agents/:encodedUrl/refresh (integration)', () => { // to complianceDb.resolveOwnerAuth (the heartbeat pattern) so the // capability probe and compliance run use the owner's saved token. it('admin refresh falls back to stored owner auth for probe and compliance (#7070)', async () => { - currentUserId = STATIC_ADMIN_USER_ID; + currentUserId = ADMIN_USER_ID; const agentUrl = ownedAgentUrl('admin-auth-fallback'); const { AgentContextDatabase } = await import('../../src/db/agent-context-db.js'); diff --git a/server/tests/unit/badge-fan-out.test.ts b/server/tests/unit/badge-fan-out.test.ts index 5504963fbe7..48a991e7861 100644 --- a/server/tests/unit/badge-fan-out.test.ts +++ b/server/tests/unit/badge-fan-out.test.ts @@ -518,4 +518,24 @@ describe('runBadgeFanOut', () => { message: 'Badge state could not be updated', }); }); + + it.each(['authorization_revoked', 'authorization_unavailable', 'authorization_provenance_missing', 'lease_lost'])( + 'preserves %s and stops badge fan-out immediately', async (code) => { + queryMock.mockResolvedValueOnce({ rows: [{ workos_organization_id: 'org_member' }], rowCount: 1 } as never); + const db = makeDb({ latestStatuses: [status('sales_broadcast_tv', 'passing')] }); + const accessFailure = Object.assign(new Error('Refresh access changed'), { code }); + vi.mocked(db.upsertBadge).mockRejectedValueOnce(accessFailure); + + await expect(runBadgeFanOut({ + complianceDb: db, + agentUrl: 'https://example.com/mcp', + declaredSpecialisms: ['sales-broadcast-tv'], + adcpVersions: ['3.0', '3.1'], + throwOnFailure: true, + })).rejects.toBe(accessFailure); + + expect(db.upsertBadge).toHaveBeenCalledOnce(); + expect(db.completeBadgeRequalification).not.toHaveBeenCalled(); + }, + ); }); diff --git a/server/tests/unit/capability-discovery-auth-classification.test.ts b/server/tests/unit/capability-discovery-auth-classification.test.ts index 3771744d67f..874dfc535ec 100644 --- a/server/tests/unit/capability-discovery-auth-classification.test.ts +++ b/server/tests/unit/capability-discovery-auth-classification.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; class MockAuthenticationRequiredError extends Error { hasOAuth = false; @@ -11,6 +11,7 @@ class MockAuthenticationRequiredError extends Error { } const getAgentInfoMock = vi.fn(); +const getAdcpCapabilitiesMock = vi.fn(); vi.mock('@adcp/sdk', () => ({ AuthenticationRequiredError: MockAuthenticationRequiredError, @@ -19,6 +20,7 @@ vi.mock('@adcp/sdk', () => ({ agent() { return { getAgentInfo: getAgentInfoMock, + getAdcpCapabilities: getAdcpCapabilitiesMock, }; } }, @@ -94,3 +96,86 @@ describe('CapabilityDiscovery auth classification', () => { expect(refreshed.discovered_tools).toEqual([]); }); }); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); + return { promise, resolve, reject }; +} + +describe('CapabilityDiscovery queued authorization checkpoints', () => { + const auth = { type: 'bearer' as const, token: 'saved-test-token' }; + const revoked = Object.assign(new Error('Access changed during the refresh'), { code: 'authorization_revoked' }); + const tools = [{ name: 'get_adcp_capabilities' }, { name: 'build_creative' }]; + + beforeEach(() => { + getAgentInfoMock.mockReset(); + getAdcpCapabilitiesMock.mockReset(); + }); + + it.each(['mcp', 'a2a'] as const)('does not start a capability call after revocation during %s discovery', async (protocol) => { + let authorized = true; + const checkpoint = async () => { if (!authorized) throw revoked; }; + const entered = deferred(); + const discovery = deferred<{ tools: typeof tools }>(); + getAgentInfoMock.mockImplementation(() => { entered.resolve(); return discovery.promise; }); + + const operation = new CapabilityDiscovery().discoverCapabilities({ ...AGENT, protocol }, auth, true, checkpoint); + const rejected = expect(operation).rejects.toBe(revoked); + await entered.promise; + authorized = false; + discovery.resolve({ tools }); + + await rejected; + expect(getAgentInfoMock).toHaveBeenCalledOnce(); + expect(getAdcpCapabilitiesMock).not.toHaveBeenCalled(); + }); + + it('preserves revocation after a pending capability response instead of using the creative fallback', async () => { + let authorized = true; + const checkpoint = async () => { if (!authorized) throw revoked; }; + const entered = deferred(); + const capabilities = deferred<{ success: boolean }>(); + getAgentInfoMock.mockResolvedValue({ tools }); + getAdcpCapabilitiesMock.mockImplementation(() => { entered.resolve(); return capabilities.promise; }); + + const operation = new CapabilityDiscovery().discoverCapabilities(AGENT, auth, true, checkpoint); + const rejected = expect(operation).rejects.toBe(revoked); + await entered.promise; + authorized = false; + capabilities.resolve({ success: false }); + + await rejected; + expect(getAdcpCapabilitiesMock).toHaveBeenCalledOnce(); + }); + + it('checks authorization again before creative fallback when the first capability call rejects', async () => { + let authorized = true; + const unavailable = Object.assign(new Error('Authorization unavailable'), { code: 'authorization_unavailable' }); + const checkpoint = async () => { if (!authorized) throw unavailable; }; + const entered = deferred(); + const capabilities = deferred(); + getAgentInfoMock.mockResolvedValue({ tools }); + getAdcpCapabilitiesMock.mockImplementation(() => { entered.resolve(); return capabilities.promise; }); + + const operation = new CapabilityDiscovery().discoverCapabilities(AGENT, auth, true, checkpoint); + const rejected = expect(operation).rejects.toBe(unavailable); + await entered.promise; + authorized = false; + capabilities.reject(new Error('Capability endpoint failed')); + + await rejected; + expect(getAdcpCapabilitiesMock).toHaveBeenCalledOnce(); + }); + + it.each(['authorization_unavailable', 'authorization_provenance_missing', 'lease_lost'])( + 'preserves %s before the first authenticated call', async (code) => { + const failure = Object.assign(new Error('Refresh access failed'), { code }); + await expect(new CapabilityDiscovery().discoverCapabilities(AGENT, auth, true, async () => { throw failure; })) + .rejects.toBe(failure); + expect(getAgentInfoMock).not.toHaveBeenCalled(); + expect(getAdcpCapabilitiesMock).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/server/tests/unit/compliance-db-authorization-guard.test.ts b/server/tests/unit/compliance-db-authorization-guard.test.ts new file mode 100644 index 00000000000..dc3ac957750 --- /dev/null +++ b/server/tests/unit/compliance-db-authorization-guard.test.ts @@ -0,0 +1,54 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { RecordComplianceRunInput } from '../../src/db/compliance-db.js'; + +const mocks = vi.hoisted(() => ({ query: vi.fn(), getClient: vi.fn() })); +vi.mock('../../src/db/client.js', () => mocks); +import { ComplianceDatabase } from '../../src/db/compliance-db.js'; + +const agentUrl = 'https://agent.example.test/mcp'; +const operations: Array<[string, (db: ComplianceDatabase) => Promise]> = [ + ['compliance run', db => db.recordComplianceRun({ agent_url: agentUrl } as RecordComplianceRunInput)], + ['badge issuance', db => db.upsertBadge({ agent_url: agentUrl, role: 'media-buy', adcp_version: '3.1', verified_specialisms: [] })], + ['badge revocation without generation', db => db.revokeBadge(agentUrl, 'media-buy', '3.1', 'Fixture revocation')], + ['badge degradation without generation', db => db.degradeBadge(agentUrl, 'media-buy', '3.1')], + ['opt-out revocation', db => db.revokeAllBadgesIfOptedOut(agentUrl, 'Fixture opt-out')], + ['requalification preparation', db => db.prepareBadgeRequalification(agentUrl, '1')], + ['requalification completion', db => db.completeBadgeRequalification(agentUrl, '1')], +]; + +describe('compliance writes with queued credential authorization', () => { + beforeEach(() => vi.resetAllMocks()); + + it.each(operations)('rolls back %s before mutation when authorization is revoked', async (_name, operation) => { + const client = { query: vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }), release: vi.fn() }; + mocks.getClient.mockResolvedValue(client); + const revoked = Object.assign(new Error('Refresh authority changed'), { code: 'authorization_revoked' }); + const beforeWrite = vi.fn().mockRejectedValue(revoked); + + await expect(operation(new ComplianceDatabase(beforeWrite))).rejects.toBe(revoked); + + expect(beforeWrite).toHaveBeenCalledExactlyOnceWith(client, agentUrl); + expect(client.query.mock.calls.map(([sql]) => sql)).toEqual(['BEGIN', 'ROLLBACK']); + expect(client.release).toHaveBeenCalledOnce(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each(['revoke', 'degrade'] as const)('keeps guarded %s writes transactional without a badge generation', async (action) => { + const client = { query: vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }), release: vi.fn() }; + mocks.getClient.mockResolvedValue(client); + const beforeWrite = vi.fn(async (transaction) => { await transaction.query('SELECT authorization_guard'); }); + const db = new ComplianceDatabase(beforeWrite); + + const result = action === 'revoke' + ? await db.revokeBadge(agentUrl, 'media-buy', '3.1', 'Fixture revocation') + : await db.degradeBadge(agentUrl, 'media-buy', '3.1'); + + expect(result).toBe(true); + const statements = client.query.mock.calls.map(([sql]) => sql as string); + expect(statements[0]).toBe('BEGIN'); + expect(statements[1]).toBe('SELECT authorization_guard'); + expect(statements.at(-1)).toBe('COMMIT'); + expect(mocks.query).not.toHaveBeenCalled(); + expect(client.release).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/tests/unit/compliance-refresh-authorization.test.ts b/server/tests/unit/compliance-refresh-authorization.test.ts new file mode 100644 index 00000000000..14f2e8b4df4 --- /dev/null +++ b/server/tests/unit/compliance-refresh-authorization.test.ts @@ -0,0 +1,184 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ComplianceRefreshRequest } from '../../src/db/compliance-refresh-requests-db.js'; + +const mocks = vi.hoisted(() => ({ getUser: vi.fn(), getWorkos: vi.fn(), query: vi.fn(), fingerprint: vi.fn() })); +vi.mock('../../src/auth/workos-client.js', () => ({ getAuthorizationEnforcementWorkos: mocks.getWorkos })); +vi.mock('../../src/db/client.js', () => ({ + query: mocks.query, + withDatabaseDeadline: (_deadline: number, operation: () => unknown) => operation(), +})); +vi.mock('../../src/db/authorization-epoch-db.js', () => ({ getAuthorizationFingerprint: mocks.fingerprint })); + +import { + authorizeComplianceRefresh, + captureComplianceRefreshAuthorization, + isRefreshAdmin, + isRefreshOwner, + resolveRefreshOwnerOrg, +} from '../../src/services/compliance-refresh-authorization.js'; + +const userRequest = { + requester_type: 'user', + requested_by_user_id: 'user_exact', + requested_by_auth_workos_user_id: 'user_exact', + authorization_fingerprint: '', + triggered_by: 'manual', + owner_org_id: null, + agent_url: 'https://agent.example.test/mcp', +} as ComplianceRefreshRequest; + +describe('queued compliance refresh authorization', () => { + beforeEach(() => { + vi.resetAllMocks(); + mocks.getWorkos.mockReturnValue({ userManagement: { getUser: mocks.getUser } }); + mocks.getUser.mockResolvedValue({ id: 'user_exact', email: 'current@example.test' }); + mocks.fingerprint.mockResolvedValue(''); + mocks.query.mockResolvedValue({ rows: [{ is_admin: true }] }); + }); + + afterEach(() => { vi.unstubAllEnvs(); }); + + it('uses only the exact live credential and current email for administrator authority', async () => { + await authorizeComplianceRefresh(userRequest); + expect(mocks.getUser).toHaveBeenCalledWith('user_exact'); + expect(mocks.query).toHaveBeenCalledWith(expect.stringContaining('working_group_memberships'), ['user_exact']); + expect(mocks.fingerprint).toHaveBeenCalledWith(['user_exact']); + }); + + it.each([401, 403, 408, 429, 500, 503])('treats WorkOS HTTP %s as unavailable without consulting local admin authority', async (status) => { + mocks.getUser.mockRejectedValue({ status }); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('treats missing WorkOS configuration as unavailable', async () => { + mocks.getWorkos.mockImplementation(() => { throw new Error('missing configuration'); }); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('bounds a stalled WorkOS body and cannot authorize from its late response', async () => { + vi.useFakeTimers(); + try { + let finish!: (credential: { id: string; email: string }) => void; + mocks.getUser.mockReturnValue(new Promise(resolve => { finish = resolve; })); + const authorization = authorizeComplianceRefresh(userRequest); + const rejected = expect(authorization).rejects.toMatchObject({ code: 'authorization_unavailable' }); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + finish({ id: 'user_exact', email: 'current@example.test' }); + await vi.advanceTimersByTimeAsync(1); + expect(mocks.query).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not authorize using a malformed WorkOS email', async () => { + mocks.getUser.mockResolvedValue({ id: 'user_exact', email: null }); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('keeps unavailable owner authorization distinct from revoked membership', async () => { + const request = { ...userRequest, triggered_by: 'owner_test' as const, owner_org_id: 'org_exact' }; + mocks.query.mockRejectedValue(new Error('database unavailable')); + await expect(authorizeComplianceRefresh(request)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + mocks.query.mockResolvedValue({ rows: [] }); + await expect(authorizeComplianceRefresh(request)).rejects.toMatchObject({ code: 'authorization_revoked' }); + expect(mocks.query.mock.calls.every(([sql]) => !sql.includes('working_group_memberships'))).toBe(true); + }); + + it('keeps unavailable administrator authority distinct from a revoked grant', async () => { + mocks.query.mockRejectedValue(new Error('administrator authority unavailable')); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + mocks.query.mockResolvedValue({ rows: [{ is_admin: false }] }); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_revoked' }); + }); + + it('keeps owner admission and polling outages retryable for the exact credential', async () => { + const principal = { id: 'user_canonical', authWorkosUserId: 'user_exact' }; + mocks.query.mockRejectedValue(new Error('database unavailable')); + await expect(resolveRefreshOwnerOrg(principal, userRequest.agent_url)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + await expect(isRefreshOwner(principal, 'org_exact', userRequest.agent_url)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.query.mock.calls.every(([, parameters]) => parameters.includes('user_exact') && !parameters.includes('user_canonical'))).toBe(true); + }); + + it('rejects unverifiable user provenance before any authority lookup', async () => { + await expect(authorizeComplianceRefresh({ ...userRequest, requested_by_auth_workos_user_id: '' })) + .rejects.toMatchObject({ code: 'authorization_provenance_missing' }); + expect(mocks.getWorkos).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('captures only the exact credential epoch after live authorization', async () => { + mocks.fingerprint.mockResolvedValue('user_exact:4'); + await expect(captureComplianceRefreshAuthorization(userRequest)).resolves.toBe('user_exact:4'); + expect(mocks.fingerprint.mock.calls.every(([ids]) => ids.length === 1 && ids[0] === 'user_exact')).toBe(true); + expect(mocks.getUser).toHaveBeenCalledExactlyOnceWith('user_exact'); + }); + + it('rejects an epoch change between admission capture and live credential lookup', async () => { + mocks.fingerprint.mockResolvedValueOnce('').mockResolvedValue('user_exact:1'); + await expect(captureComplianceRefreshAuthorization(userRequest)) + .rejects.toMatchObject({ code: 'authorization_revoked' }); + expect(mocks.getWorkos).not.toHaveBeenCalled(); + }); + + it.each(['admission', 'execution'])('rejects an epoch change during the %s WorkOS await', async (phase) => { + mocks.getUser.mockImplementationOnce(async () => { + mocks.fingerprint.mockResolvedValue('user_exact:1'); + return { id: 'user_exact', email: 'current@example.test' }; + }); + const authorize = phase === 'admission' ? captureComplianceRefreshAuthorization : authorizeComplianceRefresh; + await expect(authorize(userRequest)).rejects.toMatchObject({ code: 'authorization_revoked' }); + }); + + it('rejects a persisted epoch mismatch before looking up WorkOS or grants', async () => { + mocks.fingerprint.mockResolvedValue('user_exact:5'); + await expect(authorizeComplianceRefresh({ ...userRequest, authorization_fingerprint: 'user_exact:4' })) + .rejects.toMatchObject({ code: 'authorization_revoked' }); + expect(mocks.getWorkos).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('preserves epoch-store unavailability without falling back to cached authorization', async () => { + mocks.fingerprint.mockRejectedValue(new Error('epoch database unavailable')); + await expect(authorizeComplianceRefresh(userRequest)).rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.getWorkos).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each(['rotated-key', ''])('rejects a static administrator job after API key rotation or revocation (%s)', async (key) => { + vi.stubEnv('ADMIN_API_KEY', key); + const staticRequest = { + ...userRequest, + requester_type: 'static_admin' as const, + requested_by_user_id: null, + requested_by_auth_workos_user_id: '', + }; + await expect(captureComplianceRefreshAuthorization(staticRequest)) + .rejects.toMatchObject({ code: 'authorization_provenance_missing' }); + await expect(authorizeComplianceRefresh(staticRequest)) + .rejects.toMatchObject({ code: 'authorization_provenance_missing' }); + expect(mocks.getWorkos).not.toHaveBeenCalled(); + expect(mocks.fingerprint).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('uses current WorkOS email for independent break-glass during a membership outage', async () => { + vi.stubEnv('ADMIN_EMAILS', 'current@example.test'); + mocks.query.mockRejectedValue(new Error('membership unavailable')); + await expect(isRefreshAdmin({ id: 'user_canonical', authWorkosUserId: 'user_exact', email: 'stale@example.test' })) + .resolves.toBe(true); + expect(mocks.getUser).toHaveBeenCalledExactlyOnceWith('user_exact'); + }); + + it('does not grant break-glass from a stale session email during a membership outage', async () => { + vi.stubEnv('ADMIN_EMAILS', 'stale@example.test'); + mocks.query.mockRejectedValue(new Error('membership unavailable')); + await expect(isRefreshAdmin({ id: 'user_canonical', authWorkosUserId: 'user_exact', email: 'stale@example.test' })) + .rejects.toMatchObject({ code: 'authorization_unavailable' }); + expect(mocks.getUser).toHaveBeenCalledExactlyOnceWith('user_exact'); + }); +}); diff --git a/server/tests/unit/compliance-refresh-queue.test.ts b/server/tests/unit/compliance-refresh-queue.test.ts index 54600e91ac9..a9e3ba8c51b 100644 --- a/server/tests/unit/compliance-refresh-queue.test.ts +++ b/server/tests/unit/compliance-refresh-queue.test.ts @@ -14,6 +14,8 @@ function claimedRequest(): ClaimedComplianceRefreshRequest { owner_org_id: 'org-test', requester_type: 'user', requested_by_user_id: 'user-test', + requested_by_auth_workos_user_id: 'user-test', + authorization_fingerprint: '', triggered_by: 'owner_test', test_session_id: 'owner-refresh-test', status: 'running', @@ -63,13 +65,116 @@ describe('ComplianceRefreshQueue', () => { request.id, request.lease_token, 'authorization_provenance_missing', - 'Refresh requester authorization provenance is unavailable', + 'Authenticated credential provenance is missing; submit a new refresh', ); expect(requeueAfterFailure).not.toHaveBeenCalled(); expect(markSucceeded).not.toHaveBeenCalled(); expect(release).toHaveBeenCalledOnce(); }); + it('invalidates late probe continuations before awaiting failure persistence', async () => { + const request = claimedRequest(); + let releaseFailure!: () => void; + let failureStarted!: () => void; + const failureBarrier = new Promise(resolve => { releaseFailure = resolve; }); + const enteredFailure = new Promise(resolve => { failureStarted = resolve; }); + let assertExecutionValid!: () => void; + const db = { + claimDue: vi.fn().mockResolvedValue({ requests: [request], terminalizedExpired: 0 }), + acquireExecutionFence: vi.fn().mockResolvedValue({ isValid: () => true, release: vi.fn() }), + markFailed: vi.fn(async () => { failureStarted(); await failureBarrier; return true; }), + deleteTerminalBefore: vi.fn().mockResolvedValue(0), + }; + const work = new ComplianceRefreshQueue(async (_request, lease) => { + assertExecutionValid = lease.assertValid; + throw Object.assign(new Error('Probe timeout'), { code: 'probe_failed' }); + }, db as unknown as ComplianceRefreshRequestsDatabase).processQueue(); + await enteredFailure; + try { + expect(assertExecutionValid).toThrow(expect.objectContaining({ code: 'lease_lost' })); + } finally { + releaseFailure(); + await work; + } + expect(assertExecutionValid).toThrow(expect.objectContaining({ code: 'lease_lost' })); + }); + + it.each([false, true])('requires an atomic completion guard (registered: %s)', async (registered) => { + const request = claimedRequest(); + const guard = vi.fn().mockRejectedValue(Object.assign(new Error('revoked at the write barrier'), { + code: 'authorization_revoked', + })); + const db = { + claimDue: vi.fn().mockResolvedValue({ requests: [request], terminalizedExpired: 0 }), + acquireExecutionFence: vi.fn().mockResolvedValue({ isValid: () => true, release: vi.fn() }), + markSucceeded: vi.fn(async (_id, _token, _result, beforeWrite) => { + await beforeWrite({}); + return true; + }), + requeueAfterFailure: vi.fn().mockResolvedValue(false), + markFailed: vi.fn().mockResolvedValue(true), + deleteTerminalBefore: vi.fn().mockResolvedValue(0), + }; + await new ComplianceRefreshQueue(async (_request, lease) => { + if (registered) lease.setCompletionGuard(guard); + return { online: true }; + }, db as unknown as ComplianceRefreshRequestsDatabase).processQueue(); + + expect(db.markFailed).toHaveBeenCalledWith( + request.id, request.lease_token, + registered ? 'authorization_revoked' : 'authorization_unavailable', expect.any(String), + ); + if (registered) expect(guard).toHaveBeenCalledOnce(); + else expect(db.markSucceeded).not.toHaveBeenCalled(); + }); + + it.each([false, true])('preserves unavailable authorization when retry exhaustion is %s', async (exhausted) => { + const request = claimedRequest(); + request.attempts = exhausted ? request.max_attempts : 1; + const db = { + claimDue: vi.fn().mockResolvedValue({ requests: [request], terminalizedExpired: 0 }), + acquireExecutionFence: vi.fn().mockResolvedValue({ + isValid: () => true, + release: vi.fn().mockResolvedValue(undefined), + }), + requeueAfterFailure: vi.fn().mockResolvedValue(!exhausted), + markSucceeded: vi.fn(), + markFailed: vi.fn().mockResolvedValue(true), + deleteTerminalBefore: vi.fn().mockResolvedValue(0), + }; + const queue = new ComplianceRefreshQueue(async () => { + throw Object.assign(new Error('provider-private-diagnostic'), { code: 'authorization_unavailable' }); + }, db as unknown as ComplianceRefreshRequestsDatabase); + + await queue.processQueue(); + + const failure = [request.id, request.lease_token, 'authorization_unavailable', 'Refresh authorization is temporarily unavailable']; + expect(db.requeueAfterFailure).toHaveBeenCalledWith(...failure); + if (exhausted) expect(db.markFailed).toHaveBeenCalledWith(...failure); + else expect(db.markFailed).not.toHaveBeenCalled(); + expect(db.markSucceeded).not.toHaveBeenCalled(); + }); + + it.each(['authorization_provenance_missing', 'authorization_revoked'])('never retries confirmed %s', async (code) => { + const request = claimedRequest(); + const db = { + claimDue: vi.fn().mockResolvedValue({ requests: [request], terminalizedExpired: 0 }), + acquireExecutionFence: vi.fn().mockResolvedValue({ + isValid: () => true, + release: vi.fn().mockResolvedValue(undefined), + }), + requeueAfterFailure: vi.fn(), + markFailed: vi.fn().mockResolvedValue(true), + deleteTerminalBefore: vi.fn().mockResolvedValue(0), + }; + await new ComplianceRefreshQueue(async () => { + throw Object.assign(new Error('private-diagnostic'), { code }); + }, db as unknown as ComplianceRefreshRequestsDatabase).processQueue(); + expect(db.requeueAfterFailure).not.toHaveBeenCalled(); + expect(db.markFailed).toHaveBeenCalledWith(request.id, request.lease_token, code, expect.any(String)); + expect(JSON.stringify(db.markFailed.mock.calls)).not.toContain('private-diagnostic'); + }); + it('never persists or logs arbitrary exception codes and messages', async () => { const request = claimedRequest(); const markFailed = vi.fn().mockResolvedValue(true); diff --git a/server/tests/unit/crawler-refresh-authorization.test.ts b/server/tests/unit/crawler-refresh-authorization.test.ts new file mode 100644 index 00000000000..ef256c4bbda --- /dev/null +++ b/server/tests/unit/crawler-refresh-authorization.test.ts @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { AgentCapabilityProfile } from '../../src/capabilities.js'; +import type { AgentHealth, AgentStats } from '../../src/types.js'; +import type { ComplianceRefreshAuthorizationGuard } from '../../src/services/compliance-refresh-authorization.js'; + +const mocks = vi.hoisted(() => ({ query: vi.fn(), getClient: vi.fn() })); +vi.mock('../../src/db/client.js', async () => ({ + ...await vi.importActual('../../src/db/client.js'), + query: mocks.query, + getClient: mocks.getClient, +})); + +import { CrawlerService } from '../../src/crawler.js'; +import { AgentSnapshotDatabase } from '../../src/db/agent-snapshot-db.js'; +import { FederatedIndexService } from '../../src/federated-index.js'; + +const agentUrl = 'https://agent.example.test/mcp'; +const profile = { + agent_url: agentUrl, + protocol: 'mcp', + discovered_tools: [], + last_discovered: '2026-09-13T00:00:00.000Z', +} as unknown as AgentCapabilityProfile; +const health: AgentHealth = { online: true, checked_at: '2026-09-13T00:00:00.000Z' }; +const revoked = Object.assign(new Error('Refresh authorization changed'), { code: 'authorization_revoked' }); + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +function fixture(knownType = 'unknown') { + let authorized = true; + const client = { + query: vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }), + release: vi.fn(), + }; + mocks.getClient.mockResolvedValue(client); + const authorization: ComplianceRefreshAuthorizationGuard = { + checkpoint: vi.fn(async () => { if (!authorized) throw revoked; }), + beforeWrite: vi.fn(async (transaction, url) => { + if (!authorized) throw revoked; + expect(url).toBe(agentUrl); + await transaction.query('SELECT authorization_guard'); + }), + }; + const capabilityDiscovery = { + discoverCapabilities: vi.fn().mockResolvedValue(profile), + inferTypeFromProfile: vi.fn().mockReturnValue('sales'), + }; + const healthChecker = { + checkHealth: vi.fn().mockResolvedValue(health), + getStats: vi.fn().mockResolvedValue({}), + }; + const federatedIndex = new FederatedIndexService(); + vi.spyOn(federatedIndex, 'listAllAgents').mockResolvedValue([ + { url: agentUrl, type: knownType, protocol: 'mcp', name: 'Fixture agent' } as never, + ]); + const crawler = Object.assign(Object.create(CrawlerService.prototype) as CrawlerService, { + getPausedAgentUrls: vi.fn().mockResolvedValue(new Set()), + capabilityDiscovery, + healthChecker, + federatedIndex, + snapshotDb: new AgentSnapshotDatabase(), + }); + return { crawler, authorization, client, capabilityDiscovery, healthChecker, revoke: () => { authorized = false; } }; +} + +describe('queued refresh authorization inside the crawler', () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + }); + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('checks authorization before sending the first credential-bearing probe', async () => { + const { crawler, authorization, capabilityDiscovery, revoke } = fixture(); + revoke(); + await expect(crawler.refreshSingleAgent(agentUrl, { authorization })).rejects.toBe(revoked); + expect(capabilityDiscovery.discoverCapabilities).not.toHaveBeenCalled(); + expect(mocks.getClient).not.toHaveBeenCalled(); + }); + + it('stops after revocation during capability discovery before health, stats, or writes', async () => { + const { crawler, authorization, capabilityDiscovery, healthChecker, revoke } = fixture(); + const entered = deferred(); + const discovery = deferred(); + capabilityDiscovery.discoverCapabilities.mockImplementation(() => { + entered.resolve(); + return discovery.promise; + }); + const operation = crawler.refreshSingleAgent(agentUrl, { authorization }); + const rejected = expect(operation).rejects.toBe(revoked); + await entered.promise; + revoke(); + discovery.resolve(profile); + await rejected; + expect(healthChecker.checkHealth).not.toHaveBeenCalled(); + expect(healthChecker.getStats).not.toHaveBeenCalled(); + expect(mocks.getClient).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each(['health', 'stats'] as const)('stops canonical writes when authorization changes during %s', async (phase) => { + const { crawler, authorization, healthChecker, revoke } = fixture(); + const entered = deferred(); + const response = deferred(); + healthChecker[phase === 'health' ? 'checkHealth' : 'getStats'].mockImplementation(() => { + entered.resolve(); + return response.promise; + }); + const operation = crawler.refreshSingleAgent(agentUrl, { authorization }); + const rejected = expect(operation).rejects.toBe(revoked); + await entered.promise; + revoke(); + response.resolve(phase === 'health' ? health : {}); + await rejected; + expect(mocks.getClient).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('rejects revocation after the last checkpoint inside the write transaction', async () => { + const { crawler, authorization, client } = fixture(); + vi.mocked(authorization.beforeWrite).mockRejectedValue(revoked); + await expect(crawler.refreshSingleAgent(agentUrl, { authorization })).rejects.toBe(revoked); + expect(client.query.mock.calls.map(([sql]) => sql)).toEqual(['BEGIN', 'ROLLBACK']); + expect(client.release).toHaveBeenCalledOnce(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it('rejects a capability profile for a different agent before further probes or writes', async () => { + const { crawler, authorization, capabilityDiscovery, healthChecker } = fixture(); + capabilityDiscovery.discoverCapabilities.mockResolvedValue({ ...profile, agent_url: 'https://other.example.test/mcp' }); + await expect(crawler.refreshSingleAgent(agentUrl, { authorization })).rejects.toMatchObject({ code: 'probe_failed' }); + expect(healthChecker.checkHealth).not.toHaveBeenCalled(); + expect(healthChecker.getStats).not.toHaveBeenCalled(); + expect(mocks.getClient).not.toHaveBeenCalled(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each(['unknown', 'creative'])('keeps snapshots and %s type handling on the guarded transaction', async (knownType) => { + const { crawler, authorization, client } = fixture(knownType); + await expect(crawler.refreshSingleAgent(agentUrl, { authorization })).resolves.toMatchObject({ + online: true, + type_promoted: knownType === 'unknown', + }); + expect(authorization.beforeWrite).toHaveBeenCalledExactlyOnceWith(client, agentUrl); + const statements = client.query.mock.calls.map(([sql]) => sql as string); + expect(statements[0]).toBe('BEGIN'); + expect(statements[1]).toBe('SELECT authorization_guard'); + expect(statements).toEqual(expect.arrayContaining([ + expect.stringContaining('INSERT INTO agent_capabilities_snapshot'), + expect.stringContaining('INSERT INTO agent_health_snapshot'), + expect.stringContaining(knownType === 'unknown' ? 'UPDATE discovered_agents' : 'INSERT INTO type_reclassification_log'), + ])); + expect(statements.at(-1)).toBe('COMMIT'); + expect(client.release).toHaveBeenCalledOnce(); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each(['agent_capabilities_snapshot', 'type_reclassification_log'])('does not swallow a guarded %s write failure', async (table) => { + const { crawler, authorization, client } = fixture('creative'); + const failure = Object.assign(new Error('Transaction connection terminated'), { code: '08006' }); + client.query.mockImplementation(async (sql: string) => { + if (sql.includes(`INSERT INTO ${table}`)) throw failure; + return { rows: [], rowCount: 1 }; + }); + await expect(crawler.refreshSingleAgent(agentUrl, { authorization })).rejects.toBe(failure); + expect(client.query).toHaveBeenCalledWith('ROLLBACK'); + expect(client.query).not.toHaveBeenCalledWith('COMMIT'); + expect(client.release).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/tests/unit/resolve-user-agent-auth.test.ts b/server/tests/unit/resolve-user-agent-auth.test.ts index 87710314d4e..53db144685f 100644 --- a/server/tests/unit/resolve-user-agent-auth.test.ts +++ b/server/tests/unit/resolve-user-agent-auth.test.ts @@ -43,6 +43,32 @@ describe('resolveUserAgentAuth', () => { const call = () => resolveUserAgentAuth(db as unknown as AgentContextDatabase, ORG, URL, logger); + it.each(['authorization_revoked', 'authorization_unavailable'])('stops before OAuth secrets after an awaited context lookup: %s', async (code) => { + let release!: () => void; + let entered!: () => void; + const enteredContext = new Promise(resolve => { entered = resolve; }); + const contextBarrier = new Promise(resolve => { release = resolve; }); + db.getAuthInfoByOrgAndUrl.mockResolvedValue(null); + db.getByOrgAndUrl.mockImplementation(async () => { + entered(); + await contextBarrier; + return { id: 'ctx_guarded', has_oauth_token: true }; + }); + let revoked = false; + const failure = Object.assign(new Error('authorization changed'), { code }); + const work = resolveUserAgentAuth(db as unknown as AgentContextDatabase, ORG, URL, logger, async () => { + if (revoked) throw failure; + }); + const result = expect(work).rejects.toBe(failure); + await enteredContext; + revoked = true; + release(); + await result; + expect(db.getOAuthTokensByOrgAndUrl).not.toHaveBeenCalled(); + expect(db.getOAuthClient).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + }); + it('canonicalizes LoopMe-style path and trailing-slash variants before every lookup', async () => { const variant = 'HTTPS://PLATFORM.LOOPME.AI/MCP/SELLER/'; const canonical = 'https://platform.loopme.ai/mcp/seller'; diff --git a/server/tests/unit/workos-client-timeout.test.ts b/server/tests/unit/workos-client-timeout.test.ts index 420363f1f44..e0225db70ac 100644 --- a/server/tests/unit/workos-client-timeout.test.ts +++ b/server/tests/unit/workos-client-timeout.test.ts @@ -62,6 +62,17 @@ describe('WorkOS client request budgets', () => { }); }); + it('bounds exact credential enforcement to five seconds without SDK retries', async () => { + const { getAuthorizationEnforcementWorkos } = await import('../../src/auth/workos-client.js'); + expect(getAuthorizationEnforcementWorkos()).toBe(getAuthorizationEnforcementWorkos()); + expect(mocks.constructWorkOS).toHaveBeenCalledTimes(1); + expect(mocks.constructWorkOS).toHaveBeenCalledWith('sk_test_timeout', { + clientId: 'client_test_timeout', + timeout: 5_000, + maxRetries: 0, + }); + }); + it('makes one SDK attempt when retries are disabled', async () => { const { WorkOS } = await vi.importActual( '@workos-inc/node',