diff --git a/.changeset/board-index-releases-reads.md b/.changeset/board-index-releases-reads.md new file mode 100644 index 00000000..0339b304 --- /dev/null +++ b/.changeset/board-index-releases-reads.md @@ -0,0 +1,9 @@ +--- +"@shivaedev/work-board": patch +"@shivaedev/work-fleet": patch +--- + +### Fixed + +- `@shivaedev/work-board`: Free each Board index read once a rebuild replaces it. Every rebuilt index kept the read before it alive, so a long-running Board server on a frequently changing board grew its heap until Node ran out of memory. +- `@shivaedev/work-fleet`: The Fleet daemon no longer runs out of memory on a frequently changing board, because the Board index it reads no longer keeps every earlier read alive. diff --git a/packages/work-board/src/server/search/releasedReads.spec.ts b/packages/work-board/src/server/search/releasedReads.spec.ts new file mode 100644 index 00000000..a8a55c06 --- /dev/null +++ b/packages/work-board/src/server/search/releasedReads.spec.ts @@ -0,0 +1,19 @@ +import { expect } from "vitest"; +import { indexedFiles, indexStory } from "#test/indexStory.ts"; + +indexStory.it( + "the index keeps the read it last built, and a rebuild after the next change frees it once the reader has let go of it", + [indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "---\nid: b\n---\n# Beta" })], + function* ({ editorWrites, readerLetsGoOfRead, readerReadsIndex, releasedReadsInMemory, watcherReports }) { + yield* editorWrites("a.md", "---\nid: a\n---\n# First edit"); + yield* watcherReports("a.md"); + yield* readerLetsGoOfRead("the first-edit read"); + expect(yield* releasedReadsInMemory()).toEqual(["the first-edit read"]); + + yield* editorWrites("a.md", "---\nid: a\n---\n# Second edit"); + yield* watcherReports("a.md"); + yield* readerReadsIndex(); + + expect(yield* releasedReadsInMemory()).toEqual([]); + }, +); diff --git a/packages/work-board/src/server/search/snapshot.ts b/packages/work-board/src/server/search/snapshot.ts index 2a90ceeb..cae18527 100644 --- a/packages/work-board/src/server/search/snapshot.ts +++ b/packages/work-board/src/server/search/snapshot.ts @@ -1,4 +1,4 @@ -import { Clock, Effect, type FileSystem, Option, type Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect"; +import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect"; import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import type { MetadataDocument } from "#integration/metadata.ts"; import type { Indexing } from "#rpc/Indexing.ts"; @@ -122,7 +122,8 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file])); const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled); const enriched = enrich(next, model); - const services = yield* Effect.context(); + // Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot. + const services = (yield* Effect.context()).pipe(Context.pick(FileSystem.FileSystem, Path.Path)); const snapshot: Snapshot = { archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)), documents, diff --git a/packages/work-board/src/test-support/collectGarbage.ts b/packages/work-board/src/test-support/collectGarbage.ts new file mode 100644 index 00000000..e9d423bc --- /dev/null +++ b/packages/work-board/src/test-support/collectGarbage.ts @@ -0,0 +1,13 @@ +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { Effect } from "effect"; + +let gc: (() => void) | undefined; + +export const collectGarbage = Effect.sync(() => { + if (gc === undefined) { + setFlagsFromString("--expose-gc"); + gc = runInNewContext("gc"); + } + gc?.(); +}); diff --git a/packages/work-board/src/test-support/indexComparison.ts b/packages/work-board/src/test-support/indexComparison.ts new file mode 100644 index 00000000..c10d261a --- /dev/null +++ b/packages/work-board/src/test-support/indexComparison.ts @@ -0,0 +1,36 @@ +import { Order } from "effect"; +import type { Snapshot } from "#server/search/snapshot.ts"; + +// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them. +export function comparable(value: unknown): unknown { + if (value instanceof Map) { + return [...value].map(comparable); + } + if (value instanceof Set) { + return [...value].map(comparable); + } + if (Array.isArray(value)) { + return value.map(comparable); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([, field]) => field !== undefined) + .map(([key, field]) => [key, comparable(field)]), + ); + } + return value; +} + +export function kept(before: Snapshot, after: Snapshot) { + return after.documents + .filter((document) => { + const previous = before.documents.find((candidate) => candidate.file === document.file); + function entries(snapshot: Snapshot) { + return snapshot.entries.filter((entry) => entry.file === document.file); + } + return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry); + }) + .map((document) => document.file) + .sort(Order.String); +} diff --git a/packages/work-board/src/test-support/indexStory.ts b/packages/work-board/src/test-support/indexStory.ts index 426411e3..c9a8d62c 100644 --- a/packages/work-board/src/test-support/indexStory.ts +++ b/packages/work-board/src/test-support/indexStory.ts @@ -8,7 +8,9 @@ import { inboxItems } from "#server/attention/inbox.ts"; import { searchMatch } from "#server/search/match.ts"; import { type Snapshot, searchSnapshot } from "#server/search/snapshot.ts"; import { type Folder, folder } from "./board.ts"; +import { collectGarbage } from "./collectGarbage.ts"; import { incrementalIndex } from "./incrementalIndex.ts"; +import { comparable, kept } from "./indexComparison.ts"; import { indexState } from "./indexState.ts"; interface IndexEngine { @@ -19,6 +21,7 @@ interface IndexEngine { lastRead?: Snapshot; notes: Folder; read?: Effect.Success>; + readonly released: Map>; unreadable?: string; } @@ -41,40 +44,6 @@ function advancingListing(engine: IndexEngine, index: Effect.Success addFile)) }; } -// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them. -function comparable(value: unknown): unknown { - if (value instanceof Map) { - return [...value].map(comparable); - } - if (value instanceof Set) { - return [...value].map(comparable); - } - if (Array.isArray(value)) { - return value.map(comparable); - } - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value) - .filter(([, field]) => field !== undefined) - .map(([key, field]) => [key, comparable(field)]), - ); - } - return value; -} - -function kept(before: Snapshot, after: Snapshot) { - return after.documents - .filter((document) => { - const previous = before.documents.find((candidate) => candidate.file === document.file); - function entries(snapshot: Snapshot) { - return snapshot.entries.filter((entry) => entry.file === document.file); - } - return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry); - }) - .map((document) => document.file) - .sort(Order.String); -} - export const indexStory = storyKit({ after: { *project(engine) { @@ -100,7 +69,7 @@ export const indexStory = storyKit({ Effect.sync(() => folder({})), (acquired) => Effect.sync(acquired.remove), ); - const engine: IndexEngine = { addDuringListing: false, notes }; + const engine: IndexEngine = { addDuringListing: false, notes, released: new Map() }; return engine; }, inspect: (engine) => ({ blocked: [...(engine.index?.blocked ?? [])], reads: engine.index?.reads, root: engine.notes.root }), @@ -227,6 +196,13 @@ export const indexStory = storyKit({ story.tell("the reader counts the path and directory checks of the read"); return indexed(engine).index.checks(); }), + readerLetsGoOfRead: (name: string) => { + // Told before the read: the story's call sites would otherwise keep the read in memory. + story.tell(`the reader reads the index as ${name} and lets go of that read`); + return Effect.map(current(), (snapshot) => { + engine.released.set(name, new WeakRef(snapshot)); + }); + }, readerReadsIndex: () => Effect.map(current(), (snapshot) => { story.tell("the reader reads the index"); @@ -243,6 +219,12 @@ export const indexStory = storyKit({ indexed(engine).index.blocked.clear(); indexed(engine).index.reads.length = 0; }), + releasedReadsInMemory: () => + Effect.gen(function* () { + story.tell("garbage collection runs and the reader lists the reads it let go of that are still in memory"); + yield* collectGarbage; + return [...engine.released].filter(([, read]) => read.deref() !== undefined).map(([name]) => name); + }), revisionOfRead: () => Effect.gen(function* () { story.tell("the reader compares the read's revision with the index's current revision");