diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 1d249ff6758..68eb1a0257b 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -48,7 +48,7 @@ * `device=all`) still has to normalize every row in its window; the index makes the * selection exact, not sublinear. */ -import { Database } from "bun:sqlite"; +import { constants as sqliteConstants, Database } from "bun:sqlite"; import { randomUUID } from "node:crypto"; import { chmodSync, existsSync } from "node:fs"; import { getConfigDir } from "../config"; @@ -87,6 +87,8 @@ const MAX_REFRESH_ATTEMPTS = 3; // asynchronously instead of parking the event loop behind another writer. A single ingest batch // or publish holds the write lock for a few milliseconds. const SQLITE_BUSY_TIMEOUT_MS = 250; +/** Bound on the WAL kept between checkpoints; it is truncated to empty on the last close. */ +const RETAINED_WAL_LIMIT_BYTES = 4 * 1024 * 1024; const BATCH_BUSY_RETRIES = 3; const BATCH_BUSY_RETRY_MS = 5; const BUSY_RETRY_DELAYS_MS = [10, 25, 50] as const; @@ -280,6 +282,8 @@ interface IndexMeta { let ingestFlight: { path: string; promise: Promise } | null = null; const ingestPassesByPath = new Map(); const rebuildRequestedPaths = new Set(); +/** Index paths whose kept sidecars this process has already registered for uninstall. */ +const sidecarOwnershipChecked = new Set(); let testHooks: UsageFilterIndexTestHooks | null = null; export function usageFilterIndexPath(configDir: string): string { @@ -498,6 +502,52 @@ function ensureSchema(db: Database, classification: "fresh" | "ours"): void { recreateSchema(db, { onlyIfStale: true }); } +/** + * Keep the WAL and shared-memory sidecars after the last connection closes. + * + * SQLite's default deletes both on the final close, so every filtered query created and removed two + * entries in the config directory. On Windows that churn is not free: the config directory's ACL + * memo is keyed on its change time, so the next config load re-ran icacls over the directory, the + * inherited-ACE rewrite moved usage.jsonl's ChangeTime, and both the usage aggregate and this + * index then rebuilt from the whole ledger. macOS's system SQLite already persists WAL; this makes + * every platform behave that way. Read-only connections never delete the sidecars, so only the + * connections that can write set it, and only after the file classified as this projection: a + * foreign WAL database keeps SQLite's default and leaves nothing behind. + * + * The writer also checkpoints with TRUNCATE before it closes (see refreshUsageFilterIndex), so a + * clean close leaves the kept WAL empty on every SQLite build: a large rebuild does not leave its + * bytes on disk, and no stale frames remain to replay into whatever file later sits at this path. + * The size limit bounds the WAL between checkpoints. + */ +function keepWalSidecars(db: Database): void { + db.fileControl(sqliteConstants.SQLITE_FCNTL_PERSIST_WAL, 1); + db.exec(`PRAGMA journal_size_limit = ${RETAINED_WAL_LIMIT_BYTES}`); +} + +/** + * Register the WAL sidecars that now persist beside this projection. + * + * Creation registers the sidecar names up front, but an existing file skips that: a 0-byte + * projection left by a crash between the private create and the first open, or one whose names + * were never recorded. Its sidecars now persist and would otherwise survive uninstall. Called only + * after the file classified as this projection and WAL mode is on, so every sidecar present is + * this database's own. Sidecars appear only once SQLite has used the WAL, so the path is + * remembered only when both exist and were offered for registration; until then each open checks. + */ +function registerKeptSidecars(dir: string, path: string): void { + if (sidecarOwnershipChecked.has(path)) return; + // The same crash leaves the projection file itself unrecorded; it is this projection now. + recordOwnedConfigPath(dir, path); + let present = 0; + for (const suffix of ["-wal", "-shm"] as const) { + const sidecar = `${path}${suffix}`; + if (!existsSync(sidecar)) continue; + recordOwnedConfigPath(dir, sidecar); + present += 1; + } + if (present === 2) sidecarOwnershipChecked.add(path); +} + function openIndexForWrite(dir: string, path: string): Database { const existed = existsSync(path); preparePrivateSqliteDatabase(path); @@ -508,6 +558,7 @@ function openIndexForWrite(dir: string, path: string): Database { db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`); const classification = classifyDatabase(db); if (classification === "foreign") throw new UsageFilterIndexForeignError(); + keepWalSidecars(db); if (!existed) { // Register only names this module is about to create; an existing sidecar may belong to // something else and is never adopted. @@ -522,6 +573,7 @@ function openIndexForWrite(dir: string, path: string): Database { hardenSecretPath(path, { required: false }); } db.exec("PRAGMA journal_mode = WAL"); + registerKeptSidecars(dir, path); ensureSchema(db, classification); return db; } catch (error) { @@ -550,6 +602,7 @@ function openIndexForRead(path: string): Database { const db = new Database(path, { create: false }); try { if (classifyDatabase(db) !== "ours") throw new UsageFilterIndexForeignError(); + keepWalSidecars(db); return db; } catch (inner) { try { db.close(); } catch { /* already closed */ } @@ -1077,6 +1130,10 @@ async function refreshUsageFilterIndex( } return await refreshWithRetry(db, dir, path); } finally { + // A reader in another process can still pin the WAL; then this checkpoint is partial (SQLite + // reports that as a result row, not an error) and a later writer's TRUNCATE empties it once + // no reader pins it. The close itself never empties a kept WAL that still has frames. + try { db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); } catch { /* the close below still runs */ } db.close(); } })(); @@ -1407,5 +1464,6 @@ export function resetUsageFilterIndexForTests(): void { ingestFlight = null; ingestPassesByPath.clear(); rebuildRequestedPaths.clear(); + sidecarOwnershipChecked.clear(); testHooks = null; } diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index e84f806021d..51a2fd97ffd 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -44,6 +44,20 @@ observed during a row read re-verifies the selected bytes before publishing them offset that no longer frames a complete row drops the projection instead of serving a partial answer. `usage.jsonl` stays authoritative and is never truncated or rewritten. +The index keeps its `-wal` and `-shm` sidecars after the last connection closes +(`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write, set only once the file classified +as this projection). The writer runs `PRAGMA wal_checkpoint(TRUNCATE)` before it closes and a +`journal_size_limit` is set, so a clean close leaves the kept WAL empty; while a reader in another +process pins it, the WAL keeps its frames until a later writer's checkpoint. Creation registers the +sidecar names up front; an existing projection whose names were never recorded (a 0-byte file left +by a crash before the first open) registers them on the first open after they exist. SQLite's +default deletes them on the final close, which changed the config directory on +every filtered query. On Windows a changed +config directory re-runs its ACL harden, whose inherited-ACE rewrite moves `usage.jsonl`'s +ChangeTime, and both the usage aggregate and this index then rebuild from the whole ledger. +`tests/usage/usage-filter-index.test.ts` asserts that filtered queries after the first leave the +directory's entries and times unchanged. + The reader also receives the snapshot's revision, boundary offset and trailing digest. When the revision moved since the snapshot it re-verifies that boundary before trusting any offset, so a rewrite between the index refresh and the row read becomes a rebuild rather than a silently @@ -93,9 +107,10 @@ table without the data tables, and a schema another process committed first is a verified. An outdated schema is recreated only after re-checking it under the same lock, so two processes that both saw it stale cannot drop each other's fresh tables. -The projection is created 0600 through the shared private-SQLite path, its WAL and shared-memory -sidecars are registered for uninstall only when this module creates them, and a file at that path -that is not this projection is never modified. A passive management read opens an existing +The projection is created 0600 through the shared private-SQLite path. It and its WAL and +shared-memory sidecars are registered for uninstall when this module creates them, or, beside an +existing file that classified as this projection, once they appear; a file at that path that is +not this projection is never registered or modified. A passive management read opens an existing projection read-only, requires this projection's full table set rather than a coincidental meta table, and only uses it when it is current for the exact ledger revision; otherwise it streams, and it never creates, writes or registers anything. Any other index-side failure -- unwritable home, diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index f1ade3f2d73..0804d4e2fd1 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -6,6 +6,7 @@ import { existsSync, mkdtempSync, readFileSync, + readdirSync, renameSync, rmSync, statSync, @@ -15,6 +16,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { resetAppOwnedMemoryForTests } from "../../src/lib/app-owned-memory"; +import { initializeConfigOwnership } from "../../src/lib/config-ownership"; import { runPassiveRead } from "../../src/lib/passive-read"; import { getFilteredUsageAggregate, @@ -1126,6 +1128,72 @@ describe("filtered usage index availability", () => { } }); + test("filtered queries after the first leave the config directory's entries unchanged", async () => { + // On Windows a changed config directory re-runs its ACL harden, which moves usage.jsonl's + // ChangeTime and forces whole-ledger rebuilds; SQLite deleting its WAL sidecars on every + // close changed the directory on each filtered query. + // Discriminates only on stock SQLite (Linux, Windows): macOS's system SQLite already keeps + // the sidecars, so a local macOS pass is not evidence for this guard. + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + const first = await getFilteredUsageAggregate({ provider: "openai" }); + const observeHome = () => { + const stat = statSync(testDir); + return { entries: readdirSync(testDir).sort(), mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs }; + }; + const before = observeHome(); + const other = await getFilteredUsageAggregate({ provider: "kimi" }); + // An append makes the refresh open the index for writing again. + appendRows([{ ...syntheticRow(99), requestId: "late" }]); + const appended = await getFilteredUsageAggregate({ provider: "openai" }); + expect(observeHome()).toEqual(before); + for (const result of [first, other, appended]) expect(result.filterIndex).toBeDefined(); + // The append really went through a writable refresh, not a retained answer. + expect(appended.filterIndex?.totalRows).toBe(21); + // The kept WAL is empty after the writer's clean close. + expect(statSync(`${usageFilterIndexPath(testDir)}-wal`).size).toBe(0); + }); + + test("a projection whose sidecars are not in the uninstall manifest is repaired on the next write", async () => { + // Sidecars now persist, so any that exist beside this projection must be owned, or uninstall + // leaves them behind. + expect(initializeConfigOwnership(testDir)).toBe(true); + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + await getFilteredUsageAggregate({ provider: "openai" }); + const indexPath = usageFilterIndexPath(testDir); + const manifestPath = join(testDir, ".opencodex-uninstall.json"); + const owned = () => (JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }).paths; + const sidecars = ["usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }; + writeFileSync(manifestPath, JSON.stringify({ ...manifest, paths: manifest.paths.filter(path => !sidecars.includes(path)) }, null, 2)); + for (const suffix of ["-wal", "-shm"]) rmSync(`${indexPath}${suffix}`, { force: true }); + resetUsageAggregateCacheForTests(); + resetAppOwnedMemoryForTests(); + expect(owned().filter(path => sidecars.includes(path))).toEqual([]); + + appendRows([{ ...syntheticRow(99), requestId: "after-upgrade" }]); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + expect(owned().filter(path => sidecars.includes(path)).sort()).toEqual([...sidecars].sort()); + }); + + test("a 0-byte projection left by a crash and its sidecars are registered once they appear", async () => { + // A crash between the private create and the first open leaves an empty file, so the next open + // sees an existing projection and skips creation-time registration. The sidecars only appear + // once SQLite uses the WAL, after this open checked, so a later open in the same process must + // still register them. + expect(initializeConfigOwnership(testDir)).toBe(true); + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + const indexPath = usageFilterIndexPath(testDir); + writeFileSync(indexPath, "", { mode: 0o600 }); + const manifestPath = join(testDir, ".opencodex-uninstall.json"); + const projection = ["usage-filter-index.sqlite", "usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; + const ownedSidecars = () => (JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }).paths + .filter(path => projection.includes(path)).sort(); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + appendRows([{ ...syntheticRow(99), requestId: "after-crash" }]); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + expect(ownedSidecars()).toEqual([...projection].sort()); + }); + test("a passive read never creates or writes index files", async () => { const rows = Array.from({ length: 5 }, (_, index) => syntheticRow(index)); writeRows(rows);