From 4d705ef96efe78d8dfdb6b8972bc77a48fc7b01d Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 02:15:09 +0000 Subject: [PATCH 1/2] fix(server): retry a failed docs index and log what /ready is waiting on Production deploys have failed since #9072: in each bluegreen rollout one green machine never passed /ready within Fly's 10-minute wait_timeout and Fly rolled back (deploys of c925886c82 and 954eb9db41). /ready waits on web chat, which waits on Addie's knowledge index, and that left two gaps: - If initializeDocsIndex() threw, knowledge search stayed unready until "a later caller" retried, and after boot nothing calls it again, so one failed attempt stranded the machine. It now retries on its own with backoff (5s, 15s, 30s, then every 60s) until it succeeds. - The failure was logged as { error }, which the logger does not serialize, so the reason never appeared. It is now logged as { err } with memory usage. /ready now logs which checks block it (database, reporting, chat) with the database and reporting errors and, for chat, which part of its knowledge index is missing; once per change and at most every 60s while unchanged. Fly does not keep probe bodies, so this is how the next stuck rollout names its blocker. The 3.3-rc docs snapshot (#9071, #9072) took the docs index from 5 to 6 versions: locally, under the web process's 2 GB heap limit, peak heap during indexing went from 488 MB to 703 MB and indexing time from 12s to 19s. That is the likely trigger, but the stuck machine's own logs were lost at rollback, so this change makes the next attempt recover or report. Co-Authored-By: Claude Opus 5.5 --- server/src/addie/mcp/knowledge-search.ts | 48 +++++++++++++++++-- server/src/http.ts | 37 +++++++++++++- server/src/routes/addie-chat.ts | 6 +++ .../knowledge-search-initialization.test.ts | 44 ++++++++++++++++- 4 files changed, 127 insertions(+), 8 deletions(-) diff --git a/server/src/addie/mcp/knowledge-search.ts b/server/src/addie/mcp/knowledge-search.ts index 5bad3b3830d..b000e9c8ccf 100644 --- a/server/src/addie/mcp/knowledge-search.ts +++ b/server/src/addie/mcp/knowledge-search.ts @@ -100,6 +100,41 @@ function decodeGetDocCursor(value: string): GetDocCursor | null { let initialized = false; let initializationPromise: Promise | null = null; +/** + * Backoff between automatic retries of an incomplete initialization. Web chat + * readiness (/ready) waits on this index, and nothing else calls initialize + * again after boot, so without a retry one failed attempt would leave the + * machine unready for good. The last delay repeats until it succeeds. + */ +const RETRY_DELAYS_MS = [5_000, 15_000, 30_000, 60_000]; +let retryAttempt = 0; +let retryTimer: ReturnType | null = null; + +function scheduleRetry(): number { + const delay = RETRY_DELAYS_MS[Math.min(retryAttempt, RETRY_DELAYS_MS.length - 1)]; + retryAttempt += 1; + if (!retryTimer) { + retryTimer = setTimeout(() => { + retryTimer = null; + void initializeKnowledgeSearch(); + }, delay); + retryTimer.unref?.(); + } + return delay; +} + +/** Cancel a pending automatic retry. For tests and shutdown. */ +export function stopKnowledgeSearchRetry(): void { + if (retryTimer) clearTimeout(retryTimer); + retryTimer = null; + retryAttempt = 0; +} + +/** Which parts of knowledge search are ready, for readiness diagnostics. */ +export function getKnowledgeReadiness(): { ready: boolean; docs: boolean; externalRepos: boolean; retryAttempts: number } { + return { ready: initialized, docs: isDocsIndexReady(), externalRepos: isExternalReposReady(), retryAttempts: retryAttempt }; +} + /** * Initialize knowledge search * - Indexes docs from filesystem @@ -129,7 +164,7 @@ export function initializeKnowledgeSearch(): Promise { 'Addie: Docs index ready' ); } catch (error) { - logger.warn({ error }, 'Addie: Failed to index docs'); + logger.warn({ err: error, memory: process.memoryUsage() }, 'Addie: Failed to index docs'); } } @@ -149,14 +184,17 @@ export function initializeKnowledgeSearch(): Promise { ); } } catch (error) { - logger.warn({ error }, 'Addie: Failed to index external repos'); + logger.warn({ err: error }, 'Addie: Failed to index external repos'); } initialized = isDocsIndexReady() && isExternalReposReady(); - if (!initialized) { + if (initialized) { + stopKnowledgeSearchRetry(); + } else { + const retryInMs = scheduleRetry(); logger.warn( - { docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady() }, - 'Addie: Knowledge search initialization incomplete; a later caller will retry', + { docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady(), attempt: retryAttempt, retryInMs }, + 'Addie: Knowledge search initialization incomplete; retrying', ); } })().finally(() => { diff --git a/server/src/http.ts b/server/src/http.ts index 6d5b76e81c6..28d206acc3e 100644 --- a/server/src/http.ts +++ b/server/src/http.ts @@ -105,7 +105,7 @@ import { createAdminRouter } from "./routes/admin.js"; import { createAdminInsightsRouter } from "./routes/admin-insights.js"; import { createAddieAdminRouter } from "./routes/addie-admin.js"; import { createSecretariatAdminRouter } from "./routes/secretariat-admin.js"; -import { createAddieChatRouter, isWebChatReady } from "./routes/addie-chat.js"; +import { createAddieChatRouter, isWebChatReady, webChatReadiness } from "./routes/addie-chat.js"; import { createTavusRouter } from "./routes/tavus.js"; import { createSiChatRoutes } from "./routes/si-chat.js"; import { sendAccountLinkedMessage, invalidateMemberContextCache, isAddieBoltReady } from "./addie/index.js"; @@ -816,6 +816,11 @@ let consecutiveDbHealthFailures = 0; let dbHealthAlerted = false; const HEALTH_DB_ALERT_THRESHOLD = 3; const HEALTH_DB_SLOW_LOG_MS = 1000; +/** How often an unchanged /ready failure is logged again. Fly hides probe + * bodies, so this log is how a stuck deploy names its blocker. */ +const READY_BLOCKED_LOG_INTERVAL_MS = 60_000; +let lastReadyBlockers = ''; +let lastReadyBlockedLogAt = 0; /** * Validate slug format and check against reserved keywords @@ -3088,11 +3093,15 @@ export class HTTPServer { checks.addie = isAddieBoltReady(); checks.mcp = isMCPServerReady(); checks.chat = isWebChatReady(); + let reportingError: string | null = null; if (req.path === '/ready') { try { const reporting = getTrainingGcsReporting(); if (reporting) checks.reporting = await reporting.probe(); - } catch { checks.reporting = false; } + } catch (reportingErr) { + checks.reporting = false; + reportingError = reportingErr instanceof Error ? reportingErr.message : String(reportingErr); + } } // A listening socket and a reachable database do not mean a new web @@ -3103,6 +3112,30 @@ export class HTTPServer { const ready = checks.database && (req.path !== '/ready' || (checks.reporting !== false && (!chatRequired || checks.chat))); const status = ready ? "ok" : "unavailable"; + if (req.path === '/ready') { + const blockers = [ + ...(!checks.database ? ['database'] : []), + ...(checks.reporting === false ? ['reporting'] : []), + ...(chatRequired && !checks.chat ? ['chat'] : []), + ]; + const key = blockers.join(','); + const now = Date.now(); + if (blockers.length > 0 && (key !== lastReadyBlockers || now - lastReadyBlockedLogAt >= READY_BLOCKED_LOG_INTERVAL_MS)) { + logger.warn({ + blockers, + checks, + ...(dbError && { dbError }), + ...(reportingError && { reportingError }), + ...(blockers.includes('chat') && { chat: webChatReadiness() }), + uptimeSec: Math.round(process.uptime()), + heapUsedMb: Math.round(process.memoryUsage().heapUsed / 1048576), + }, 'Readiness probe failing'); + lastReadyBlockedLogAt = now; + } else if (blockers.length === 0 && lastReadyBlockers !== '') { + logger.info({ uptimeSec: Math.round(process.uptime()) }, 'Readiness probe passing'); + } + lastReadyBlockers = key; + } const body: Record = { status, checks, diff --git a/server/src/routes/addie-chat.ts b/server/src/routes/addie-chat.ts index 67019941e23..024dfa70894 100644 --- a/server/src/routes/addie-chat.ts +++ b/server/src/routes/addie-chat.ts @@ -59,6 +59,7 @@ import { import { matchRuleIdFromMessage } from "../addie/home/builders/rules/prompt-rules.js"; import { recordPromptClicked } from "../db/addie-prompt-telemetry-db.js"; import { + getKnowledgeReadiness, isKnowledgeReady, initializeKnowledgeSearch, KNOWLEDGE_TOOLS, @@ -629,6 +630,11 @@ export function isWebChatReady(): boolean { return initialized && claudeClient !== null && isKnowledgeReady(); } +/** What web chat readiness is waiting on, for /ready diagnostics. */ +export function webChatReadiness(): Record { + return { toolsRegistered: initialized, client: claudeClient !== null, knowledge: getKnowledgeReadiness() }; +} + /** Get the chat client, initializing it first if necessary. */ export async function getChatClaudeClient(): Promise { if (!initialized) { diff --git a/server/tests/unit/knowledge-search-initialization.test.ts b/server/tests/unit/knowledge-search-initialization.test.ts index b9e301e14b3..a3694e78865 100644 --- a/server/tests/unit/knowledge-search-initialization.test.ts +++ b/server/tests/unit/knowledge-search-initialization.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ docsReady: false, @@ -55,6 +55,48 @@ describe('knowledge search initialization', () => { mocks.initializeExternalRepos.mockReset(); }); + afterEach(async () => { + const { stopKnowledgeSearchRetry } = await import('../../src/addie/mcp/knowledge-search.js'); + stopKnowledgeSearchRetry(); + vi.useRealTimers(); + }); + + it('retries a failed docs index on its own, so readiness recovers without another caller', async () => { + vi.useFakeTimers(); + const { initializeKnowledgeSearch, isKnowledgeReady, getKnowledgeReadiness } = await import( + '../../src/addie/mcp/knowledge-search.js' + ); + mocks.initializeDocsIndex + .mockRejectedValueOnce(new Error('docs index failed at boot')) + .mockRejectedValueOnce(new Error('docs index failed again')) + .mockImplementationOnce(async () => { + mocks.docsReady = true; + }); + mocks.initializeExternalRepos.mockImplementation(async () => { + mocks.externalReady = true; + }); + + await initializeKnowledgeSearch(); + expect(isKnowledgeReady()).toBe(false); + expect(getKnowledgeReadiness()).toMatchObject({ ready: false, docs: false, externalRepos: true, retryAttempts: 1 }); + + await vi.advanceTimersByTimeAsync(5_000); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2); + expect(isKnowledgeReady()).toBe(false); + + // The second retry waits longer. + await vi.advanceTimersByTimeAsync(14_999); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3); + expect(isKnowledgeReady()).toBe(true); + expect(getKnowledgeReadiness()).toMatchObject({ ready: true, retryAttempts: 0 }); + + // Nothing further is scheduled once ready. + await vi.advanceTimersByTimeAsync(120_000); + expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3); + }); + it('deduplicates concurrent calls and retries an incomplete initialization', async () => { const { initializeKnowledgeSearch, isKnowledgeReady } = await import( '../../src/addie/mcp/knowledge-search.js' From e56f364e5ec452ce1e539cbbcfc54c9a89ae362b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 02:23:28 +0000 Subject: [PATCH 2/2] chore(addie): refresh the tool-surface inventory hash for addie-chat.ts The /ready diagnostics change added webChatReadiness() to server/src/routes/addie-chat.ts, so its source hash in the generated inventory moved. Regenerated with npm run build:addie-tools; npm run test:addie-tools passes. Co-Authored-By: Claude Opus 5.5 --- .../src/addie/generated/tool-surface-inventory.generated.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/addie/generated/tool-surface-inventory.generated.json b/server/src/addie/generated/tool-surface-inventory.generated.json index 26878cbae3d..ad6c667a631 100644 --- a/server/src/addie/generated/tool-surface-inventory.generated.json +++ b/server/src/addie/generated/tool-surface-inventory.generated.json @@ -71,7 +71,7 @@ "server/src/addie/prompt-assembly.ts": "6c002ae5a52cdd26e13f424f6e00757dff163b3b273a1b4ab069fdb86f4f375f", "server/src/addie/register-baseline-tools.ts": "52893917fa19fa8904e1667857a7c4bfc2dc90bd3341e60d1ff45a40400d17aa", "server/src/addie/bolt-app.ts": "0d5ef7a6d58e04a8ff769dd186e1ff1bf0bf1583190d86127f66a411183eb980", - "server/src/routes/addie-chat.ts": "ac07f24d422d3e55cc03bdc7a7025f6501acdedb57e68c7dbfca21f747508649", + "server/src/routes/addie-chat.ts": "451df91d67d1c61ae7cd43d85f82b73decf10a00e4e80301554443a62386fba7", "server/src/routes/tavus.ts": "b0669bd8d6fb6caba858954a939fbd4330387c9b9f8bb8090cf8e294f7fa9773", "server/src/addie/email-conversation-handler.ts": "d4f5090d6f71937fff4ff8774ff516ca814be4bd7b23566b366fa037dcc642f3", "server/src/mcp/chat-tool.ts": "a55a9412eb57ce2560b3cdbf2a63c1c1e9fb71361ff1690fc70d5b37b8158258",