diff --git a/server/src/addie/generated/tool-surface-inventory.generated.json b/server/src/addie/generated/tool-surface-inventory.generated.json index 26878cbae3..ad6c667a63 100644 --- a/server/src/addie/generated/tool-surface-inventory.generated.json +++ b/server/src/addie/generated/tool-surface-inventory.generated.json @@ -71,7 +71,7 @@ "server/src/addie/prompt-assembly.ts": "6c002ae5a52cdd26e13f424f6e00757dff163b3b273a1b4ab069fdb86f4f375f", "server/src/addie/register-baseline-tools.ts": "52893917fa19fa8904e1667857a7c4bfc2dc90bd3341e60d1ff45a40400d17aa", "server/src/addie/bolt-app.ts": "0d5ef7a6d58e04a8ff769dd186e1ff1bf0bf1583190d86127f66a411183eb980", - "server/src/routes/addie-chat.ts": "ac07f24d422d3e55cc03bdc7a7025f6501acdedb57e68c7dbfca21f747508649", + "server/src/routes/addie-chat.ts": "451df91d67d1c61ae7cd43d85f82b73decf10a00e4e80301554443a62386fba7", "server/src/routes/tavus.ts": "b0669bd8d6fb6caba858954a939fbd4330387c9b9f8bb8090cf8e294f7fa9773", "server/src/addie/email-conversation-handler.ts": "d4f5090d6f71937fff4ff8774ff516ca814be4bd7b23566b366fa037dcc642f3", "server/src/mcp/chat-tool.ts": "a55a9412eb57ce2560b3cdbf2a63c1c1e9fb71361ff1690fc70d5b37b8158258", diff --git a/server/src/addie/mcp/knowledge-search.ts b/server/src/addie/mcp/knowledge-search.ts index 5bad3b3830..b000e9c8cc 100644 --- a/server/src/addie/mcp/knowledge-search.ts +++ b/server/src/addie/mcp/knowledge-search.ts @@ -100,6 +100,41 @@ function decodeGetDocCursor(value: string): GetDocCursor | null { let initialized = false; let initializationPromise: Promise | null = null; +/** + * Backoff between automatic retries of an incomplete initialization. Web chat + * readiness (/ready) waits on this index, and nothing else calls initialize + * again after boot, so without a retry one failed attempt would leave the + * machine unready for good. The last delay repeats until it succeeds. + */ +const RETRY_DELAYS_MS = [5_000, 15_000, 30_000, 60_000]; +let retryAttempt = 0; +let retryTimer: ReturnType | null = null; + +function scheduleRetry(): number { + const delay = RETRY_DELAYS_MS[Math.min(retryAttempt, RETRY_DELAYS_MS.length - 1)]; + retryAttempt += 1; + if (!retryTimer) { + retryTimer = setTimeout(() => { + retryTimer = null; + void initializeKnowledgeSearch(); + }, delay); + retryTimer.unref?.(); + } + return delay; +} + +/** Cancel a pending automatic retry. For tests and shutdown. */ +export function stopKnowledgeSearchRetry(): void { + if (retryTimer) clearTimeout(retryTimer); + retryTimer = null; + retryAttempt = 0; +} + +/** Which parts of knowledge search are ready, for readiness diagnostics. */ +export function getKnowledgeReadiness(): { ready: boolean; docs: boolean; externalRepos: boolean; retryAttempts: number } { + return { ready: initialized, docs: isDocsIndexReady(), externalRepos: isExternalReposReady(), retryAttempts: retryAttempt }; +} + /** * Initialize knowledge search * - Indexes docs from filesystem @@ -129,7 +164,7 @@ export function initializeKnowledgeSearch(): Promise { 'Addie: Docs index ready' ); } catch (error) { - logger.warn({ error }, 'Addie: Failed to index docs'); + logger.warn({ err: error, memory: process.memoryUsage() }, 'Addie: Failed to index docs'); } } @@ -149,14 +184,17 @@ export function initializeKnowledgeSearch(): Promise { ); } } catch (error) { - logger.warn({ error }, 'Addie: Failed to index external repos'); + logger.warn({ err: error }, 'Addie: Failed to index external repos'); } initialized = isDocsIndexReady() && isExternalReposReady(); - if (!initialized) { + if (initialized) { + stopKnowledgeSearchRetry(); + } else { + const retryInMs = scheduleRetry(); logger.warn( - { docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady() }, - 'Addie: Knowledge search initialization incomplete; a later caller will retry', + { docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady(), attempt: retryAttempt, retryInMs }, + 'Addie: Knowledge search initialization incomplete; retrying', ); } })().finally(() => { diff --git a/server/src/http.ts b/server/src/http.ts index 6d5b76e81c..28d206acc3 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -105,7 +105,7 @@ import { createAdminRouter } from "./routes/admin.js"; import { createAdminInsightsRouter } from "./routes/admin-insights.js"; import { createAddieAdminRouter } from "./routes/addie-admin.js"; import { createSecretariatAdminRouter } from "./routes/secretariat-admin.js"; -import { createAddieChatRouter, isWebChatReady } from "./routes/addie-chat.js"; +import { createAddieChatRouter, isWebChatReady, webChatReadiness } from "./routes/addie-chat.js"; import { createTavusRouter } from "./routes/tavus.js"; import { createSiChatRoutes } from "./routes/si-chat.js"; import { sendAccountLinkedMessage, invalidateMemberContextCache, isAddieBoltReady } from "./addie/index.js"; @@ -816,6 +816,11 @@ let consecutiveDbHealthFailures = 0; let dbHealthAlerted = false; const HEALTH_DB_ALERT_THRESHOLD = 3; const HEALTH_DB_SLOW_LOG_MS = 1000; +/** How often an unchanged /ready failure is logged again. Fly hides probe + * bodies, so this log is how a stuck deploy names its blocker. */ +const READY_BLOCKED_LOG_INTERVAL_MS = 60_000; +let lastReadyBlockers = ''; +let lastReadyBlockedLogAt = 0; /** * Validate slug format and check against reserved keywords @@ -3088,11 +3093,15 @@ export class HTTPServer { checks.addie = isAddieBoltReady(); checks.mcp = isMCPServerReady(); checks.chat = isWebChatReady(); + let reportingError: string | null = null; if (req.path === '/ready') { try { const reporting = getTrainingGcsReporting(); if (reporting) checks.reporting = await reporting.probe(); - } catch { checks.reporting = false; } + } catch (reportingErr) { + checks.reporting = false; + reportingError = reportingErr instanceof Error ? reportingErr.message : String(reportingErr); + } } // A listening socket and a reachable database do not mean a new web @@ -3103,6 +3112,30 @@ export class HTTPServer { const ready = checks.database && (req.path !== '/ready' || (checks.reporting !== false && (!chatRequired || checks.chat))); const status = ready ? "ok" : "unavailable"; + if (req.path === '/ready') { + const blockers = [ + ...(!checks.database ? ['database'] : []), + ...(checks.reporting === false ? ['reporting'] : []), + ...(chatRequired && !checks.chat ? ['chat'] : []), + ]; + const key = blockers.join(','); + const now = Date.now(); + if (blockers.length > 0 && (key !== lastReadyBlockers || now - lastReadyBlockedLogAt >= READY_BLOCKED_LOG_INTERVAL_MS)) { + logger.warn({ + blockers, + checks, + ...(dbError && { dbError }), + ...(reportingError && { reportingError }), + ...(blockers.includes('chat') && { chat: webChatReadiness() }), + uptimeSec: Math.round(process.uptime()), + heapUsedMb: Math.round(process.memoryUsage().heapUsed / 1048576), + }, 'Readiness probe failing'); + lastReadyBlockedLogAt = now; + } else if (blockers.length === 0 && lastReadyBlockers !== '') { + logger.info({ uptimeSec: Math.round(process.uptime()) }, 'Readiness probe passing'); + } + lastReadyBlockers = key; + } const body: Record = { status, checks, diff --git a/server/src/routes/addie-chat.ts b/server/src/routes/addie-chat.ts index 67019941e2..024dfa7089 100644 --- a/server/src/routes/addie-chat.ts +++ b/server/src/routes/addie-chat.ts @@ -59,6 +59,7 @@ import { import { matchRuleIdFromMessage } from "../addie/home/builders/rules/prompt-rules.js"; import { recordPromptClicked } from "../db/addie-prompt-telemetry-db.js"; import { + getKnowledgeReadiness, isKnowledgeReady, initializeKnowledgeSearch, KNOWLEDGE_TOOLS, @@ -629,6 +630,11 @@ export function isWebChatReady(): boolean { return initialized && claudeClient !== null && isKnowledgeReady(); } +/** What web chat readiness is waiting on, for /ready diagnostics. */ +export function webChatReadiness(): Record { + return { toolsRegistered: initialized, client: claudeClient !== null, knowledge: getKnowledgeReadiness() }; +} + /** Get the chat client, initializing it first if necessary. */ export async function getChatClaudeClient(): Promise { if (!initialized) { diff --git a/server/tests/unit/knowledge-search-initialization.test.ts b/server/tests/unit/knowledge-search-initialization.test.ts index b9e301e14b..a3694e7886 100644 --- a/server/tests/unit/knowledge-search-initialization.test.ts +++ b/server/tests/unit/knowledge-search-initialization.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ docsReady: false, @@ -55,6 +55,48 @@ describe('knowledge search initialization', () => { mocks.initializeExternalRepos.mockReset(); }); + afterEach(async () => { + const { stopKnowledgeSearchRetry } = await import('../../src/addie/mcp/knowledge-search.js'); + stopKnowledgeSearchRetry(); + vi.useRealTimers(); + }); + + it('retries a failed docs index on its own, so readiness recovers without another caller', async () => { + vi.useFakeTimers(); + const { initializeKnowledgeSearch, isKnowledgeReady, getKnowledgeReadiness } = await import( + '../../src/addie/mcp/knowledge-search.js' + ); + mocks.initializeDocsIndex + .mockRejectedValueOnce(new Error('docs index failed at boot')) + .mockRejectedValueOnce(new Error('docs index failed again')) + .mockImplementationOnce(async () => { + mocks.docsReady = true; + }); + mocks.initializeExternalRepos.mockImplementation(async () => { + mocks.externalReady = true; + }); + + await initializeKnowledgeSearch(); + expect(isKnowledgeReady()).toBe(false); + expect(getKnowledgeReadiness()).toMatchObject({ ready: false, docs: false, externalRepos: true, retryAttempts: 1 }); + + await vi.advanceTimersByTimeAsync(5_000); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2); + expect(isKnowledgeReady()).toBe(false); + + // The second retry waits longer. + await vi.advanceTimersByTimeAsync(14_999); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3); + expect(isKnowledgeReady()).toBe(true); + expect(getKnowledgeReadiness()).toMatchObject({ ready: true, retryAttempts: 0 }); + + // Nothing further is scheduled once ready. + await vi.advanceTimersByTimeAsync(120_000); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3); + }); + it('deduplicates concurrent calls and retries an incomplete initialization', async () => { const { initializeKnowledgeSearch, isKnowledgeReady } = await import( '../../src/addie/mcp/knowledge-search.js'