Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
"server/src/addie/prompt-assembly.ts": "6c002ae5a52cdd26e13f424f6e00757dff163b3b273a1b4ab069fdb86f4f375f",
"server/src/addie/register-baseline-tools.ts": "52893917fa19fa8904e1667857a7c4bfc2dc90bd3341e60d1ff45a40400d17aa",
"server/src/addie/bolt-app.ts": "0d5ef7a6d58e04a8ff769dd186e1ff1bf0bf1583190d86127f66a411183eb980",
"server/src/routes/addie-chat.ts": "ac07f24d422d3e55cc03bdc7a7025f6501acdedb57e68c7dbfca21f747508649",
"server/src/routes/addie-chat.ts": "451df91d67d1c61ae7cd43d85f82b73decf10a00e4e80301554443a62386fba7",
"server/src/routes/tavus.ts": "b0669bd8d6fb6caba858954a939fbd4330387c9b9f8bb8090cf8e294f7fa9773",
"server/src/addie/email-conversation-handler.ts": "d4f5090d6f71937fff4ff8774ff516ca814be4bd7b23566b366fa037dcc642f3",
"server/src/mcp/chat-tool.ts": "a55a9412eb57ce2560b3cdbf2a63c1c1e9fb71361ff1690fc70d5b37b8158258",
Expand Down
48 changes: 43 additions & 5 deletions server/src/addie/mcp/knowledge-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,41 @@ function decodeGetDocCursor(value: string): GetDocCursor | null {
let initialized = false;
let initializationPromise: Promise<void> | null = null;

/**
* Backoff between automatic retries of an incomplete initialization. Web chat
* readiness (/ready) waits on this index, and nothing else calls initialize
* again after boot, so without a retry one failed attempt would leave the
* machine unready for good. The last delay repeats until it succeeds.
*/
const RETRY_DELAYS_MS = [5_000, 15_000, 30_000, 60_000];
let retryAttempt = 0;
let retryTimer: ReturnType<typeof setTimeout> | null = null;

function scheduleRetry(): number {
const delay = RETRY_DELAYS_MS[Math.min(retryAttempt, RETRY_DELAYS_MS.length - 1)];
retryAttempt += 1;
if (!retryTimer) {
retryTimer = setTimeout(() => {
retryTimer = null;
void initializeKnowledgeSearch();
}, delay);
retryTimer.unref?.();
}
return delay;
}

/** Cancel a pending automatic retry. For tests and shutdown. */
export function stopKnowledgeSearchRetry(): void {
if (retryTimer) clearTimeout(retryTimer);
retryTimer = null;
retryAttempt = 0;
}

/** Which parts of knowledge search are ready, for readiness diagnostics. */
export function getKnowledgeReadiness(): { ready: boolean; docs: boolean; externalRepos: boolean; retryAttempts: number } {
return { ready: initialized, docs: isDocsIndexReady(), externalRepos: isExternalReposReady(), retryAttempts: retryAttempt };
}

/**
* Initialize knowledge search
* - Indexes docs from filesystem
Expand Down Expand Up @@ -129,7 +164,7 @@ export function initializeKnowledgeSearch(): Promise<void> {
'Addie: Docs index ready'
);
} catch (error) {
logger.warn({ error }, 'Addie: Failed to index docs');
logger.warn({ err: error, memory: process.memoryUsage() }, 'Addie: Failed to index docs');
}
}

Expand All @@ -149,14 +184,17 @@ export function initializeKnowledgeSearch(): Promise<void> {
);
}
} catch (error) {
logger.warn({ error }, 'Addie: Failed to index external repos');
logger.warn({ err: error }, 'Addie: Failed to index external repos');
}

initialized = isDocsIndexReady() && isExternalReposReady();
if (!initialized) {
if (initialized) {
stopKnowledgeSearchRetry();
} else {
const retryInMs = scheduleRetry();
logger.warn(
{ docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady() },
'Addie: Knowledge search initialization incomplete; a later caller will retry',
{ docsReady: isDocsIndexReady(), externalReposReady: isExternalReposReady(), attempt: retryAttempt, retryInMs },
'Addie: Knowledge search initialization incomplete; retrying',
);
}
})().finally(() => {
Expand Down
37 changes: 35 additions & 2 deletions server/src/http.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ import { createAdminRouter } from "./routes/admin.js";
import { createAdminInsightsRouter } from "./routes/admin-insights.js";
import { createAddieAdminRouter } from "./routes/addie-admin.js";
import { createSecretariatAdminRouter } from "./routes/secretariat-admin.js";
import { createAddieChatRouter, isWebChatReady } from "./routes/addie-chat.js";
import { createAddieChatRouter, isWebChatReady, webChatReadiness } from "./routes/addie-chat.js";
import { createTavusRouter } from "./routes/tavus.js";
import { createSiChatRoutes } from "./routes/si-chat.js";
import { sendAccountLinkedMessage, invalidateMemberContextCache, isAddieBoltReady } from "./addie/index.js";
Expand Down Expand Up @@ -816,6 +816,11 @@ let consecutiveDbHealthFailures = 0;
let dbHealthAlerted = false;
const HEALTH_DB_ALERT_THRESHOLD = 3;
const HEALTH_DB_SLOW_LOG_MS = 1000;
/** How often an unchanged /ready failure is logged again. Fly hides probe
* bodies, so this log is how a stuck deploy names its blocker. */
const READY_BLOCKED_LOG_INTERVAL_MS = 60_000;
let lastReadyBlockers = '';
let lastReadyBlockedLogAt = 0;

/**
* Validate slug format and check against reserved keywords
Expand Down Expand Up @@ -3088,11 +3093,15 @@ export class HTTPServer {
checks.addie = isAddieBoltReady();
checks.mcp = isMCPServerReady();
checks.chat = isWebChatReady();
let reportingError: string | null = null;
if (req.path === '/ready') {
try {
const reporting = getTrainingGcsReporting();
if (reporting) checks.reporting = await reporting.probe();
} catch { checks.reporting = false; }
} catch (reportingErr) {
checks.reporting = false;
reportingError = reportingErr instanceof Error ? reportingErr.message : String(reportingErr);
}
}

// A listening socket and a reachable database do not mean a new web
Expand All @@ -3103,6 +3112,30 @@ export class HTTPServer {
const ready = checks.database && (req.path !== '/ready'
|| (checks.reporting !== false && (!chatRequired || checks.chat)));
const status = ready ? "ok" : "unavailable";
if (req.path === '/ready') {
const blockers = [
...(!checks.database ? ['database'] : []),
...(checks.reporting === false ? ['reporting'] : []),
...(chatRequired && !checks.chat ? ['chat'] : []),
];
const key = blockers.join(',');
const now = Date.now();
if (blockers.length > 0 && (key !== lastReadyBlockers || now - lastReadyBlockedLogAt >= READY_BLOCKED_LOG_INTERVAL_MS)) {
logger.warn({
blockers,
checks,
...(dbError && { dbError }),
...(reportingError && { reportingError }),
...(blockers.includes('chat') && { chat: webChatReadiness() }),
uptimeSec: Math.round(process.uptime()),
heapUsedMb: Math.round(process.memoryUsage().heapUsed / 1048576),
}, 'Readiness probe failing');
lastReadyBlockedLogAt = now;
} else if (blockers.length === 0 && lastReadyBlockers !== '') {
logger.info({ uptimeSec: Math.round(process.uptime()) }, 'Readiness probe passing');
}
lastReadyBlockers = key;
}
const body: Record<string, unknown> = {
status,
checks,
Expand Down
6 changes: 6 additions & 0 deletions server/src/routes/addie-chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import {
import { matchRuleIdFromMessage } from "../addie/home/builders/rules/prompt-rules.js";
import { recordPromptClicked } from "../db/addie-prompt-telemetry-db.js";
import {
getKnowledgeReadiness,
isKnowledgeReady,
initializeKnowledgeSearch,
KNOWLEDGE_TOOLS,
Expand Down Expand Up @@ -629,6 +630,11 @@ export function isWebChatReady(): boolean {
return initialized && claudeClient !== null && isKnowledgeReady();
}

/** What web chat readiness is waiting on, for /ready diagnostics. */
export function webChatReadiness(): Record<string, unknown> {
return { toolsRegistered: initialized, client: claudeClient !== null, knowledge: getKnowledgeReadiness() };
}

/** Get the chat client, initializing it first if necessary. */
export async function getChatClaudeClient(): Promise<AddieClaudeClient> {
if (!initialized) {
Expand Down
44 changes: 43 additions & 1 deletion server/tests/unit/knowledge-search-initialization.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

const mocks = vi.hoisted(() => ({
docsReady: false,
Expand Down Expand Up @@ -55,6 +55,48 @@ describe('knowledge search initialization', () => {
mocks.initializeExternalRepos.mockReset();
});

afterEach(async () => {
const { stopKnowledgeSearchRetry } = await import('../../src/addie/mcp/knowledge-search.js');
stopKnowledgeSearchRetry();
vi.useRealTimers();
});

it('retries a failed docs index on its own, so readiness recovers without another caller', async () => {
vi.useFakeTimers();
const { initializeKnowledgeSearch, isKnowledgeReady, getKnowledgeReadiness } = await import(
'../../src/addie/mcp/knowledge-search.js'
);
mocks.initializeDocsIndex
.mockRejectedValueOnce(new Error('docs index failed at boot'))
.mockRejectedValueOnce(new Error('docs index failed again'))
.mockImplementationOnce(async () => {
mocks.docsReady = true;
});
mocks.initializeExternalRepos.mockImplementation(async () => {
mocks.externalReady = true;
});

await initializeKnowledgeSearch();
expect(isKnowledgeReady()).toBe(false);
expect(getKnowledgeReadiness()).toMatchObject({ ready: false, docs: false, externalRepos: true, retryAttempts: 1 });

await vi.advanceTimersByTimeAsync(5_000);
expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2);
expect(isKnowledgeReady()).toBe(false);

// The second retry waits longer.
await vi.advanceTimersByTimeAsync(14_999);
expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(1);
expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3);
expect(isKnowledgeReady()).toBe(true);
expect(getKnowledgeReadiness()).toMatchObject({ ready: true, retryAttempts: 0 });

// Nothing further is scheduled once ready.
await vi.advanceTimersByTimeAsync(120_000);
expect(mocks.initializeDocsIndex).toHaveBeenCalledTimes(3);
});

it('deduplicates concurrent calls and retries an incomplete initialization', async () => {
const { initializeKnowledgeSearch, isKnowledgeReady } = await import(
'../../src/addie/mcp/knowledge-search.js'
Expand Down
Loading