From 9f895ed6d1e5cb4580223d638ae0177f076b7ee2 Mon Sep 17 00:00:00 2001 From: JUN Date: Mon, 9 Nov 2026 17:03:57 +0900 Subject: [PATCH 1/6] fix(usage): keep the filter index's WAL sidecars so filtered queries stop churning the config directory --- src/usage/filter-index.ts | 19 ++++++++++++++++++- structure/usage-client-attribution.md | 8 ++++++++ tests/usage/usage-filter-index.test.ts | 20 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 30db3c6e78b..2e748137a96 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -48,7 +48,7 @@ * `device=all`) still has to normalize every row in its window; the index makes the * selection exact, not sublinear. */ -import { Database } from "bun:sqlite"; +import { constants as sqliteConstants, Database } from "bun:sqlite"; import { randomUUID } from "node:crypto"; import { chmodSync, existsSync } from "node:fs"; import { getConfigDir } from "../config"; @@ -461,11 +461,27 @@ function ensureSchema(db: Database, classification: "fresh" | "ours"): void { recreateSchema(db); } +/** + * Keep the WAL and shared-memory sidecars after the last connection closes. + * + * SQLite's default deletes both on the final close, so every filtered query created and removed two + * entries in the config directory. On Windows that churn is not free: the config directory's ACL + * memo is keyed on its change time, so the next config load re-ran icacls over the directory, the + * inherited-ACE rewrite moved usage.jsonl's ChangeTime, and both the usage aggregate and this + * index then rebuilt from the whole ledger. macOS's system SQLite already persists WAL; this makes + * every platform behave that way. Read-only connections never delete the sidecars, so only the + * connections that can write set it. + */ +function keepWalSidecars(db: Database): void { + db.fileControl(sqliteConstants.SQLITE_FCNTL_PERSIST_WAL, 1); +} + function openIndexForWrite(dir: string, path: string): Database { const existed = existsSync(path); preparePrivateSqliteDatabase(path); const db = new Database(path, { create: true }); try { + keepWalSidecars(db); const classification = classifyDatabase(db); if (classification === "foreign") throw new UsageFilterIndexForeignError(); if (!existed) { @@ -510,6 +526,7 @@ function openIndexForRead(path: string): Database { if (!existsSync(path)) throw error; const db = new Database(path, { create: false }); try { + keepWalSidecars(db); if (classifyDatabase(db) !== "ours") throw new UsageFilterIndexForeignError(); return db; } catch (inner) { diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index 9a7ca1a5b8f..c053f542c0f 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -44,6 +44,14 @@ observed during a row read re-verifies the selected bytes before publishing them offset that no longer frames a complete row drops the projection instead of serving a partial answer. `usage.jsonl` stays authoritative and is never truncated or rewritten. +The index keeps its `-wal` and `-shm` sidecars after the last connection closes +(`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write). SQLite's default deletes them on +the final close, which changed the config directory on every filtered query. On Windows a changed +config directory re-runs its ACL harden, whose inherited-ACE rewrite moves `usage.jsonl`'s +ChangeTime, and both the usage aggregate and this index then rebuild from the whole ledger. +`tests/usage/usage-filter-index.test.ts` asserts that filtered queries after the first leave the +directory's entries and times unchanged. + The reader also receives the snapshot's revision, boundary offset and trailing digest. When the revision moved since the snapshot it re-verifies that boundary before trusting any offset, so a rewrite between the index refresh and the row read becomes a rebuild rather than a silently diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index 3272cbd3d51..74ec05acf01 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -6,6 +6,7 @@ import { existsSync, mkdtempSync, readFileSync, + readdirSync, renameSync, rmSync, statSync, @@ -1089,6 +1090,25 @@ describe("filtered usage index availability", () => { } }); + test("filtered queries after the first leave the config directory's entries unchanged", async () => { + // On Windows a changed config directory re-runs its ACL harden, which moves usage.jsonl's + // ChangeTime and forces whole-ledger rebuilds; SQLite deleting its WAL sidecars on every + // close changed the directory on each filtered query. + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + const first = await getFilteredUsageAggregate({ provider: "openai" }); + const observeHome = () => { + const stat = statSync(testDir); + return { entries: readdirSync(testDir).sort(), mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs }; + }; + const before = observeHome(); + const other = await getFilteredUsageAggregate({ provider: "kimi" }); + // An append makes the refresh open the index for writing again. + appendRows([{ ...syntheticRow(99), requestId: "late" }]); + const appended = await getFilteredUsageAggregate({ provider: "openai" }); + expect(observeHome()).toEqual(before); + for (const result of [first, other, appended]) expect(result.filterIndex).toBeDefined(); + }); + test("a passive read never creates or writes index files", async () => { const rows = Array.from({ length: 5 }, (_, index) => syntheticRow(index)); writeRows(rows); From d81706bd10daa0ac7d059a17d39c5e934ff3cbf1 Mon Sep 17 00:00:00 2001 From: JUN Date: Mon, 9 Nov 2026 17:30:07 +0900 Subject: [PATCH 2/6] fix(usage): keep WAL sidecars only for this projection and truncate the kept WAL on close --- src/usage/filter-index.ts | 13 ++++++++++--- structure/usage-client-attribution.md | 4 +++- tests/usage/usage-filter-index.test.ts | 2 ++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 2e748137a96..dcc80af490a 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -87,6 +87,8 @@ const MAX_REFRESH_ATTEMPTS = 3; // asynchronously instead of parking the event loop behind another writer. A single ingest batch // or publish holds the write lock for a few milliseconds. const SQLITE_BUSY_TIMEOUT_MS = 250; +/** Bound on the WAL kept between checkpoints; it is truncated to empty on the last close. */ +const RETAINED_WAL_LIMIT_BYTES = 4 * 1024 * 1024; const BATCH_BUSY_RETRIES = 3; const BATCH_BUSY_RETRY_MS = 5; const BUSY_RETRY_DELAYS_MS = [10, 25, 50] as const; @@ -470,10 +472,15 @@ function ensureSchema(db: Database, classification: "fresh" | "ours"): void { * inherited-ACE rewrite moved usage.jsonl's ChangeTime, and both the usage aggregate and this * index then rebuilt from the whole ledger. macOS's system SQLite already persists WAL; this makes * every platform behave that way. Read-only connections never delete the sidecars, so only the - * connections that can write set it. + * connections that can write set it, and only after the file classified as this projection: a + * foreign WAL database keeps SQLite's default and leaves nothing behind. + * + * A size limit makes the retained WAL truncate to empty when the last connection closes, so a + * large rebuild does not leave its WAL on disk; the file stays, the bytes do not. */ function keepWalSidecars(db: Database): void { db.fileControl(sqliteConstants.SQLITE_FCNTL_PERSIST_WAL, 1); + db.exec(`PRAGMA journal_size_limit = ${RETAINED_WAL_LIMIT_BYTES}`); } function openIndexForWrite(dir: string, path: string): Database { @@ -481,9 +488,9 @@ function openIndexForWrite(dir: string, path: string): Database { preparePrivateSqliteDatabase(path); const db = new Database(path, { create: true }); try { - keepWalSidecars(db); const classification = classifyDatabase(db); if (classification === "foreign") throw new UsageFilterIndexForeignError(); + keepWalSidecars(db); if (!existed) { // Register only names this module is about to create; an existing sidecar may belong to // something else and is never adopted. @@ -526,8 +533,8 @@ function openIndexForRead(path: string): Database { if (!existsSync(path)) throw error; const db = new Database(path, { create: false }); try { - keepWalSidecars(db); if (classifyDatabase(db) !== "ours") throw new UsageFilterIndexForeignError(); + keepWalSidecars(db); return db; } catch (inner) { try { db.close(); } catch { /* already closed */ } diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index c053f542c0f..58ff295f504 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -45,7 +45,9 @@ offset that no longer frames a complete row drops the projection instead of serv answer. `usage.jsonl` stays authoritative and is never truncated or rewritten. The index keeps its `-wal` and `-shm` sidecars after the last connection closes -(`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write). SQLite's default deletes them on +(`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write, set only once the file classified +as this projection; a `journal_size_limit` truncates the kept WAL to empty on that close). SQLite's +default deletes them on the final close, which changed the config directory on every filtered query. On Windows a changed config directory re-runs its ACL harden, whose inherited-ACE rewrite moves `usage.jsonl`'s ChangeTime, and both the usage aggregate and this index then rebuild from the whole ledger. diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index 74ec05acf01..bc73852b367 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -1107,6 +1107,8 @@ describe("filtered usage index availability", () => { const appended = await getFilteredUsageAggregate({ provider: "openai" }); expect(observeHome()).toEqual(before); for (const result of [first, other, appended]) expect(result.filterIndex).toBeDefined(); + // The append really went through a writable refresh, not a retained answer. + expect(appended.filterIndex?.totalRows).toBe(21); }); test("a passive read never creates or writes index files", async () => { From cb1588b23b177477fd99330e2a09c5f5e7b19379 Mon Sep 17 00:00:00 2001 From: JUN Date: Mon, 9 Nov 2026 23:12:09 +0900 Subject: [PATCH 3/6] fix(usage): checkpoint the kept WAL with TRUNCATE before the writer closes --- src/usage/filter-index.ts | 9 +++++++-- tests/usage/usage-filter-index.test.ts | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 48eee618a0d..2232f874a04 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -512,8 +512,10 @@ function ensureSchema(db: Database, classification: "fresh" | "ours"): void { * connections that can write set it, and only after the file classified as this projection: a * foreign WAL database keeps SQLite's default and leaves nothing behind. * - * A size limit makes the retained WAL truncate to empty when the last connection closes, so a - * large rebuild does not leave its WAL on disk; the file stays, the bytes do not. + * The writer also checkpoints with TRUNCATE before it closes (see refreshUsageFilterIndex), so a + * clean close leaves the kept WAL empty on every SQLite build: a large rebuild does not leave its + * bytes on disk, and no stale frames remain to replay into whatever file later sits at this path. + * The size limit bounds the WAL between checkpoints. */ function keepWalSidecars(db: Database): void { db.fileControl(sqliteConstants.SQLITE_FCNTL_PERSIST_WAL, 1); @@ -1101,6 +1103,9 @@ async function refreshUsageFilterIndex( } return await refreshWithRetry(db, dir, path); } finally { + // A reader in another process can still hold the WAL; then this checkpoint is partial and + // the next writer's close empties it. + try { db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); } catch { /* the close below still runs */ } db.close(); } })(); diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index 01c7b3da19b..750b1a476a4 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -1146,6 +1146,8 @@ describe("filtered usage index availability", () => { for (const result of [first, other, appended]) expect(result.filterIndex).toBeDefined(); // The append really went through a writable refresh, not a retained answer. expect(appended.filterIndex?.totalRows).toBe(21); + // The kept WAL is empty after the writer's clean close. + expect(statSync(`${usageFilterIndexPath(testDir)}-wal`).size).toBe(0); }); test("a passive read never creates or writes index files", async () => { From fcf9ebd9bac700c535beba79d62dd3c52bc60cf4 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 10 Nov 2026 01:48:27 +0900 Subject: [PATCH 4/6] fix(usage): register kept WAL sidecars beside a projection built before they persisted --- src/usage/filter-index.ts | 26 ++++++++++++++++++++++++-- structure/usage-client-attribution.md | 9 ++++++--- tests/usage/usage-filter-index.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 2232f874a04..cebe90bbf31 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -282,6 +282,8 @@ interface IndexMeta { let ingestFlight: { path: string; promise: Promise } | null = null; const ingestPassesByPath = new Map(); const rebuildRequestedPaths = new Set(); +/** Index paths whose kept sidecars this process has already registered for uninstall. */ +const sidecarOwnershipChecked = new Set(); let testHooks: UsageFilterIndexTestHooks | null = null; export function usageFilterIndexPath(configDir: string): string { @@ -522,6 +524,23 @@ function keepWalSidecars(db: Database): void { db.exec(`PRAGMA journal_size_limit = ${RETAINED_WAL_LIMIT_BYTES}`); } +/** + * Register the WAL sidecars that now persist beside this projection. + * + * A projection built before persistence deleted them on every close, so it was opened as an + * existing file with nothing on disk to register, and the sidecars it keeps from here on would + * otherwise survive uninstall. Called only after the file classified as this projection and WAL + * mode is on, so every sidecar present is this database's own. + */ +function registerKeptSidecars(dir: string, path: string): void { + if (sidecarOwnershipChecked.has(path)) return; + for (const suffix of ["-wal", "-shm"] as const) { + const sidecar = `${path}${suffix}`; + if (existsSync(sidecar)) recordOwnedConfigPath(dir, sidecar); + } + sidecarOwnershipChecked.add(path); +} + function openIndexForWrite(dir: string, path: string): Database { const existed = existsSync(path); preparePrivateSqliteDatabase(path); @@ -547,6 +566,7 @@ function openIndexForWrite(dir: string, path: string): Database { hardenSecretPath(path, { required: false }); } db.exec("PRAGMA journal_mode = WAL"); + registerKeptSidecars(dir, path); ensureSchema(db, classification); return db; } catch (error) { @@ -1103,8 +1123,9 @@ async function refreshUsageFilterIndex( } return await refreshWithRetry(db, dir, path); } finally { - // A reader in another process can still hold the WAL; then this checkpoint is partial and - // the next writer's close empties it. + // A reader in another process can still pin the WAL; then this checkpoint is partial (SQLite + // reports that as a result row, not an error) and a later writer's TRUNCATE empties it once + // no reader pins it. The close itself never empties a kept WAL that still has frames. try { db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); } catch { /* the close below still runs */ } db.close(); } @@ -1436,5 +1457,6 @@ export function resetUsageFilterIndexForTests(): void { ingestFlight = null; ingestPassesByPath.clear(); rebuildRequestedPaths.clear(); + sidecarOwnershipChecked.clear(); testHooks = null; } diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index b92012b4bb0..b4e9541b439 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -46,9 +46,12 @@ answer. `usage.jsonl` stays authoritative and is never truncated or rewritten. The index keeps its `-wal` and `-shm` sidecars after the last connection closes (`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write, set only once the file classified -as this projection; a `journal_size_limit` truncates the kept WAL to empty on that close). SQLite's -default deletes them on -the final close, which changed the config directory on every filtered query. On Windows a changed +as this projection). The writer runs `PRAGMA wal_checkpoint(TRUNCATE)` before it closes and a +`journal_size_limit` is set, so a clean close leaves the kept WAL empty; while a reader in another +process pins it, the WAL keeps its frames until a later writer's checkpoint. The sidecars are +registered for uninstall once they exist, including beside a projection built before they were +kept. SQLite's default deletes them on the final close, which changed the config directory on +every filtered query. On Windows a changed config directory re-runs its ACL harden, whose inherited-ACE rewrite moves `usage.jsonl`'s ChangeTime, and both the usage aggregate and this index then rebuild from the whole ledger. `tests/usage/usage-filter-index.test.ts` asserts that filtered queries after the first leave the diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index 750b1a476a4..b28c22116fa 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -16,6 +16,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { resetAppOwnedMemoryForTests } from "../../src/lib/app-owned-memory"; +import { initializeConfigOwnership } from "../../src/lib/config-ownership"; import { runPassiveRead } from "../../src/lib/passive-read"; import { getFilteredUsageAggregate, @@ -1131,6 +1132,8 @@ describe("filtered usage index availability", () => { // On Windows a changed config directory re-runs its ACL harden, which moves usage.jsonl's // ChangeTime and forces whole-ledger rebuilds; SQLite deleting its WAL sidecars on every // close changed the directory on each filtered query. + // Discriminates only on stock SQLite (Linux, Windows): macOS's system SQLite already keeps + // the sidecars, so a local macOS pass is not evidence for this guard. writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); const first = await getFilteredUsageAggregate({ provider: "openai" }); const observeHome = () => { @@ -1150,6 +1153,29 @@ describe("filtered usage index availability", () => { expect(statSync(`${usageFilterIndexPath(testDir)}-wal`).size).toBe(0); }); + test("sidecars of a projection built before WAL persistence are registered for uninstall", async () => { + // A projection from a build that deleted its WAL sidecars on every close has no sidecars on + // disk and none in the uninstall manifest. Once they persist they must be owned, or uninstall + // leaves them behind. + expect(initializeConfigOwnership(testDir)).toBe(true); + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + await getFilteredUsageAggregate({ provider: "openai" }); + const indexPath = usageFilterIndexPath(testDir); + const manifestPath = join(testDir, ".opencodex-uninstall.json"); + const owned = () => (JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }).paths; + const sidecars = ["usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }; + writeFileSync(manifestPath, JSON.stringify({ ...manifest, paths: manifest.paths.filter(path => !sidecars.includes(path)) }, null, 2)); + for (const suffix of ["-wal", "-shm"]) rmSync(`${indexPath}${suffix}`, { force: true }); + resetUsageAggregateCacheForTests(); + resetAppOwnedMemoryForTests(); + expect(owned().filter(path => sidecars.includes(path))).toEqual([]); + + appendRows([{ ...syntheticRow(99), requestId: "after-upgrade" }]); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + expect(owned().filter(path => sidecars.includes(path)).sort()).toEqual([...sidecars].sort()); + }); + test("a passive read never creates or writes index files", async () => { const rows = Array.from({ length: 5 }, (_, index) => syntheticRow(index)); writeRows(rows); From 45e01384d4f81eb7d7cf7c8d76251d075e1ec62d Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 10 Nov 2026 02:33:15 +0900 Subject: [PATCH 5/6] fix(usage): keep checking sidecar ownership until both sidecars exist --- src/usage/filter-index.ts | 17 +++++++++++------ structure/usage-client-attribution.md | 7 ++++--- tests/usage/usage-filter-index.test.ts | 24 +++++++++++++++++++++--- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index cebe90bbf31..0151d71d2a2 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -527,18 +527,23 @@ function keepWalSidecars(db: Database): void { /** * Register the WAL sidecars that now persist beside this projection. * - * A projection built before persistence deleted them on every close, so it was opened as an - * existing file with nothing on disk to register, and the sidecars it keeps from here on would - * otherwise survive uninstall. Called only after the file classified as this projection and WAL - * mode is on, so every sidecar present is this database's own. + * Creation registers the sidecar names up front, but an existing file skips that: a 0-byte + * projection left by a crash between the private create and the first open, or one whose names + * were never recorded. Its sidecars now persist and would otherwise survive uninstall. Called only + * after the file classified as this projection and WAL mode is on, so every sidecar present is + * this database's own. Sidecars appear only once SQLite has used the WAL, so the path is + * remembered only when both exist and were offered for registration; until then each open checks. */ function registerKeptSidecars(dir: string, path: string): void { if (sidecarOwnershipChecked.has(path)) return; + let present = 0; for (const suffix of ["-wal", "-shm"] as const) { const sidecar = `${path}${suffix}`; - if (existsSync(sidecar)) recordOwnedConfigPath(dir, sidecar); + if (!existsSync(sidecar)) continue; + recordOwnedConfigPath(dir, sidecar); + present += 1; } - sidecarOwnershipChecked.add(path); + if (present === 2) sidecarOwnershipChecked.add(path); } function openIndexForWrite(dir: string, path: string): Database { diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index b4e9541b439..85578a41d80 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -48,9 +48,10 @@ The index keeps its `-wal` and `-shm` sidecars after the last connection closes (`SQLITE_FCNTL_PERSIST_WAL` on every connection that can write, set only once the file classified as this projection). The writer runs `PRAGMA wal_checkpoint(TRUNCATE)` before it closes and a `journal_size_limit` is set, so a clean close leaves the kept WAL empty; while a reader in another -process pins it, the WAL keeps its frames until a later writer's checkpoint. The sidecars are -registered for uninstall once they exist, including beside a projection built before they were -kept. SQLite's default deletes them on the final close, which changed the config directory on +process pins it, the WAL keeps its frames until a later writer's checkpoint. Creation registers the +sidecar names up front; an existing projection whose names were never recorded (a 0-byte file left +by a crash before the first open) registers them on the first open after they exist. SQLite's +default deletes them on the final close, which changed the config directory on every filtered query. On Windows a changed config directory re-runs its ACL harden, whose inherited-ACE rewrite moves `usage.jsonl`'s ChangeTime, and both the usage aggregate and this index then rebuild from the whole ledger. diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index b28c22116fa..4e3dbfb9b01 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -1153,9 +1153,8 @@ describe("filtered usage index availability", () => { expect(statSync(`${usageFilterIndexPath(testDir)}-wal`).size).toBe(0); }); - test("sidecars of a projection built before WAL persistence are registered for uninstall", async () => { - // A projection from a build that deleted its WAL sidecars on every close has no sidecars on - // disk and none in the uninstall manifest. Once they persist they must be owned, or uninstall + test("a projection whose sidecars are not in the uninstall manifest is repaired on the next write", async () => { + // Sidecars now persist, so any that exist beside this projection must be owned, or uninstall // leaves them behind. expect(initializeConfigOwnership(testDir)).toBe(true); writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); @@ -1176,6 +1175,25 @@ describe("filtered usage index availability", () => { expect(owned().filter(path => sidecars.includes(path)).sort()).toEqual([...sidecars].sort()); }); + test("sidecars beside a 0-byte projection left by a crash are registered once they appear", async () => { + // A crash between the private create and the first open leaves an empty file, so the next open + // sees an existing projection and skips creation-time registration. The sidecars only appear + // once SQLite uses the WAL, after this open checked, so a later open in the same process must + // still register them. + expect(initializeConfigOwnership(testDir)).toBe(true); + writeRows(Array.from({ length: 20 }, (_, index) => syntheticRow(index))); + const indexPath = usageFilterIndexPath(testDir); + writeFileSync(indexPath, "", { mode: 0o600 }); + const manifestPath = join(testDir, ".opencodex-uninstall.json"); + const sidecars = ["usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; + const ownedSidecars = () => (JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }).paths + .filter(path => sidecars.includes(path)).sort(); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + appendRows([{ ...syntheticRow(99), requestId: "after-crash" }]); + expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); + expect(ownedSidecars()).toEqual([...sidecars].sort()); + }); + test("a passive read never creates or writes index files", async () => { const rows = Array.from({ length: 5 }, (_, index) => syntheticRow(index)); writeRows(rows); From 05966a2a2a432681a565bcdec50777c2fb2cc258 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 10 Nov 2026 03:13:51 +0900 Subject: [PATCH 6/6] fix(usage): register a crash-left projection file with its kept sidecars --- src/usage/filter-index.ts | 2 ++ structure/usage-client-attribution.md | 7 ++++--- tests/usage/usage-filter-index.test.ts | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/usage/filter-index.ts b/src/usage/filter-index.ts index 0151d71d2a2..68eb1a0257b 100644 --- a/src/usage/filter-index.ts +++ b/src/usage/filter-index.ts @@ -536,6 +536,8 @@ function keepWalSidecars(db: Database): void { */ function registerKeptSidecars(dir: string, path: string): void { if (sidecarOwnershipChecked.has(path)) return; + // The same crash leaves the projection file itself unrecorded; it is this projection now. + recordOwnedConfigPath(dir, path); let present = 0; for (const suffix of ["-wal", "-shm"] as const) { const sidecar = `${path}${suffix}`; diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index 85578a41d80..51a2fd97ffd 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -107,9 +107,10 @@ table without the data tables, and a schema another process committed first is a verified. An outdated schema is recreated only after re-checking it under the same lock, so two processes that both saw it stale cannot drop each other's fresh tables. -The projection is created 0600 through the shared private-SQLite path, its WAL and shared-memory -sidecars are registered for uninstall only when this module creates them, and a file at that path -that is not this projection is never modified. A passive management read opens an existing +The projection is created 0600 through the shared private-SQLite path. It and its WAL and +shared-memory sidecars are registered for uninstall when this module creates them, or, beside an +existing file that classified as this projection, once they appear; a file at that path that is +not this projection is never registered or modified. A passive management read opens an existing projection read-only, requires this projection's full table set rather than a coincidental meta table, and only uses it when it is current for the exact ledger revision; otherwise it streams, and it never creates, writes or registers anything. Any other index-side failure -- unwritable home, diff --git a/tests/usage/usage-filter-index.test.ts b/tests/usage/usage-filter-index.test.ts index 4e3dbfb9b01..0804d4e2fd1 100644 --- a/tests/usage/usage-filter-index.test.ts +++ b/tests/usage/usage-filter-index.test.ts @@ -1175,7 +1175,7 @@ describe("filtered usage index availability", () => { expect(owned().filter(path => sidecars.includes(path)).sort()).toEqual([...sidecars].sort()); }); - test("sidecars beside a 0-byte projection left by a crash are registered once they appear", async () => { + test("a 0-byte projection left by a crash and its sidecars are registered once they appear", async () => { // A crash between the private create and the first open leaves an empty file, so the next open // sees an existing projection and skips creation-time registration. The sidecars only appear // once SQLite uses the WAL, after this open checked, so a later open in the same process must @@ -1185,13 +1185,13 @@ describe("filtered usage index availability", () => { const indexPath = usageFilterIndexPath(testDir); writeFileSync(indexPath, "", { mode: 0o600 }); const manifestPath = join(testDir, ".opencodex-uninstall.json"); - const sidecars = ["usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; + const projection = ["usage-filter-index.sqlite", "usage-filter-index.sqlite-wal", "usage-filter-index.sqlite-shm"]; const ownedSidecars = () => (JSON.parse(readFileSync(manifestPath, "utf8")) as { paths: string[] }).paths - .filter(path => sidecars.includes(path)).sort(); + .filter(path => projection.includes(path)).sort(); expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); appendRows([{ ...syntheticRow(99), requestId: "after-crash" }]); expect((await getFilteredUsageAggregate({ provider: "openai" })).filterIndex).toBeDefined(); - expect(ownedSidecars()).toEqual([...sidecars].sort()); + expect(ownedSidecars()).toEqual([...projection].sort()); }); test("a passive read never creates or writes index files", async () => {