From 62b9410bf96dc102ee5ae8fb54a516a70a183100 Mon Sep 17 00:00:00 2001 From: Marvin Bitterlich <644950+marvin-bitterlich@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:04:19 +0000 Subject: [PATCH 1/2] Board index reads free the read before them Each index read kept the whole fiber context to load archived records later. That context holds the buildSnapshot stack frame, whose arguments include the previous read, so every read kept every earlier one alive and the Fleet daemon ran out of heap within minutes. Keep only FileSystem and Path instead. --- .changeset/board-index-releases-reads.md | 8 +++ .../src/server/search/releasedReads.spec.ts | 19 +++++++ .../work-board/src/server/search/snapshot.ts | 5 +- .../src/test-support/collectGarbage.ts | 13 +++++ .../src/test-support/indexComparison.ts | 36 +++++++++++++ .../work-board/src/test-support/indexStory.ts | 52 ++++++------------- 6 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 .changeset/board-index-releases-reads.md create mode 100644 packages/work-board/src/server/search/releasedReads.spec.ts create mode 100644 packages/work-board/src/test-support/collectGarbage.ts create mode 100644 packages/work-board/src/test-support/indexComparison.ts diff --git a/.changeset/board-index-releases-reads.md b/.changeset/board-index-releases-reads.md new file mode 100644 index 00000000..7e74a97b --- /dev/null +++ b/.changeset/board-index-releases-reads.md @@ -0,0 +1,8 @@ +--- +"@shivaedev/work-board": patch +"@shivaedev/work-fleet": patch +--- + +### Fixed + +- Free each Board index read once a rebuild replaces it. Every rebuilt index kept the read before it alive, so a long-running Board or Fleet daemon on a frequently changing board grew its heap until Node ran out of memory. diff --git a/packages/work-board/src/server/search/releasedReads.spec.ts b/packages/work-board/src/server/search/releasedReads.spec.ts new file mode 100644 index 00000000..a8a55c06 --- /dev/null +++ b/packages/work-board/src/server/search/releasedReads.spec.ts @@ -0,0 +1,19 @@ +import { expect } from "vitest"; +import { indexedFiles, indexStory } from "#test/indexStory.ts"; + +indexStory.it( + "the index keeps the read it last built, and a rebuild after the next change frees it once the reader has let go of it", + [indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "---\nid: b\n---\n# Beta" })], + function* ({ editorWrites, readerLetsGoOfRead, readerReadsIndex, releasedReadsInMemory, watcherReports }) { + yield* editorWrites("a.md", "---\nid: a\n---\n# First edit"); + yield* watcherReports("a.md"); + yield* readerLetsGoOfRead("the first-edit read"); + expect(yield* releasedReadsInMemory()).toEqual(["the first-edit read"]); + + yield* editorWrites("a.md", "---\nid: a\n---\n# Second edit"); + yield* watcherReports("a.md"); + yield* readerReadsIndex(); + + expect(yield* releasedReadsInMemory()).toEqual([]); + }, +); diff --git a/packages/work-board/src/server/search/snapshot.ts b/packages/work-board/src/server/search/snapshot.ts index 2a90ceeb..cae18527 100644 --- a/packages/work-board/src/server/search/snapshot.ts +++ b/packages/work-board/src/server/search/snapshot.ts @@ -1,4 +1,4 @@ -import { Clock, Effect, type FileSystem, Option, type Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect"; +import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect"; import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import type { MetadataDocument } from "#integration/metadata.ts"; import type { Indexing } from "#rpc/Indexing.ts"; @@ -122,7 +122,8 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file])); const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled); const enriched = enrich(next, model); - const services = yield* Effect.context(); + // Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot. + const services = (yield* Effect.context()).pipe(Context.pick(FileSystem.FileSystem, Path.Path)); const snapshot: Snapshot = { archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)), documents, diff --git a/packages/work-board/src/test-support/collectGarbage.ts b/packages/work-board/src/test-support/collectGarbage.ts new file mode 100644 index 00000000..e9d423bc --- /dev/null +++ b/packages/work-board/src/test-support/collectGarbage.ts @@ -0,0 +1,13 @@ +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { Effect } from "effect"; + +let gc: (() => void) | undefined; + +export const collectGarbage = Effect.sync(() => { + if (gc === undefined) { + setFlagsFromString("--expose-gc"); + gc = runInNewContext("gc"); + } + gc?.(); +}); diff --git a/packages/work-board/src/test-support/indexComparison.ts b/packages/work-board/src/test-support/indexComparison.ts new file mode 100644 index 00000000..c10d261a --- /dev/null +++ b/packages/work-board/src/test-support/indexComparison.ts @@ -0,0 +1,36 @@ +import { Order } from "effect"; +import type { Snapshot } from "#server/search/snapshot.ts"; + +// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them. +export function comparable(value: unknown): unknown { + if (value instanceof Map) { + return [...value].map(comparable); + } + if (value instanceof Set) { + return [...value].map(comparable); + } + if (Array.isArray(value)) { + return value.map(comparable); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([, field]) => field !== undefined) + .map(([key, field]) => [key, comparable(field)]), + ); + } + return value; +} + +export function kept(before: Snapshot, after: Snapshot) { + return after.documents + .filter((document) => { + const previous = before.documents.find((candidate) => candidate.file === document.file); + function entries(snapshot: Snapshot) { + return snapshot.entries.filter((entry) => entry.file === document.file); + } + return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry); + }) + .map((document) => document.file) + .sort(Order.String); +} diff --git a/packages/work-board/src/test-support/indexStory.ts b/packages/work-board/src/test-support/indexStory.ts index 426411e3..c9a8d62c 100644 --- a/packages/work-board/src/test-support/indexStory.ts +++ b/packages/work-board/src/test-support/indexStory.ts @@ -8,7 +8,9 @@ import { inboxItems } from "#server/attention/inbox.ts"; import { searchMatch } from "#server/search/match.ts"; import { type Snapshot, searchSnapshot } from "#server/search/snapshot.ts"; import { type Folder, folder } from "./board.ts"; +import { collectGarbage } from "./collectGarbage.ts"; import { incrementalIndex } from "./incrementalIndex.ts"; +import { comparable, kept } from "./indexComparison.ts"; import { indexState } from "./indexState.ts"; interface IndexEngine { @@ -19,6 +21,7 @@ interface IndexEngine { lastRead?: Snapshot; notes: Folder; read?: Effect.Success>; + readonly released: Map>; unreadable?: string; } @@ -41,40 +44,6 @@ function advancingListing(engine: IndexEngine, index: Effect.Success addFile)) }; } -// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them. -function comparable(value: unknown): unknown { - if (value instanceof Map) { - return [...value].map(comparable); - } - if (value instanceof Set) { - return [...value].map(comparable); - } - if (Array.isArray(value)) { - return value.map(comparable); - } - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value) - .filter(([, field]) => field !== undefined) - .map(([key, field]) => [key, comparable(field)]), - ); - } - return value; -} - -function kept(before: Snapshot, after: Snapshot) { - return after.documents - .filter((document) => { - const previous = before.documents.find((candidate) => candidate.file === document.file); - function entries(snapshot: Snapshot) { - return snapshot.entries.filter((entry) => entry.file === document.file); - } - return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry); - }) - .map((document) => document.file) - .sort(Order.String); -} - export const indexStory = storyKit({ after: { *project(engine) { @@ -100,7 +69,7 @@ export const indexStory = storyKit({ Effect.sync(() => folder({})), (acquired) => Effect.sync(acquired.remove), ); - const engine: IndexEngine = { addDuringListing: false, notes }; + const engine: IndexEngine = { addDuringListing: false, notes, released: new Map() }; return engine; }, inspect: (engine) => ({ blocked: [...(engine.index?.blocked ?? [])], reads: engine.index?.reads, root: engine.notes.root }), @@ -227,6 +196,13 @@ export const indexStory = storyKit({ story.tell("the reader counts the path and directory checks of the read"); return indexed(engine).index.checks(); }), + readerLetsGoOfRead: (name: string) => { + // Told before the read: the story's call sites would otherwise keep the read in memory. + story.tell(`the reader reads the index as ${name} and lets go of that read`); + return Effect.map(current(), (snapshot) => { + engine.released.set(name, new WeakRef(snapshot)); + }); + }, readerReadsIndex: () => Effect.map(current(), (snapshot) => { story.tell("the reader reads the index"); @@ -243,6 +219,12 @@ export const indexStory = storyKit({ indexed(engine).index.blocked.clear(); indexed(engine).index.reads.length = 0; }), + releasedReadsInMemory: () => + Effect.gen(function* () { + story.tell("garbage collection runs and the reader lists the reads it let go of that are still in memory"); + yield* collectGarbage; + return [...engine.released].filter(([, read]) => read.deref() !== undefined).map(([name]) => name); + }), revisionOfRead: () => Effect.gen(function* () { story.tell("the reader compares the read's revision with the index's current revision"); From d39c45a981b3f08afb17beeeab278e13cb016397 Mon Sep 17 00:00:00 2001 From: Marvin Bitterlich <644950+marvin-bitterlich@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:06:50 +0000 Subject: [PATCH 2/2] Attribute the memory fix notes to each package --- .changeset/board-index-releases-reads.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.changeset/board-index-releases-reads.md b/.changeset/board-index-releases-reads.md index 7e74a97b..0339b304 100644 --- a/.changeset/board-index-releases-reads.md +++ b/.changeset/board-index-releases-reads.md @@ -5,4 +5,5 @@ ### Fixed -- Free each Board index read once a rebuild replaces it. Every rebuilt index kept the read before it alive, so a long-running Board or Fleet daemon on a frequently changing board grew its heap until Node ran out of memory. +- `@shivaedev/work-board`: Free each Board index read once a rebuild replaces it. Every rebuilt index kept the read before it alive, so a long-running Board server on a frequently changing board grew its heap until Node ran out of memory. +- `@shivaedev/work-fleet`: The Fleet daemon no longer runs out of memory on a frequently changing board, because the Board index it reads no longer keeps every earlier read alive.