Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/board-index-releases-reads.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
"@shivaedev/work-board": patch
"@shivaedev/work-fleet": patch
---

### Fixed

- `@shivaedev/work-board`: Free each Board index read once a rebuild replaces it. Every rebuilt index kept the read before it alive, so a long-running Board server on a frequently changing board grew its heap until Node ran out of memory.
- `@shivaedev/work-fleet`: The Fleet daemon no longer runs out of memory on a frequently changing board, because the Board index it reads no longer keeps every earlier read alive.
19 changes: 19 additions & 0 deletions packages/work-board/src/server/search/releasedReads.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { expect } from "vitest";
import { indexedFiles, indexStory } from "#test/indexStory.ts";

indexStory.it(
"the index keeps the read it last built, and a rebuild after the next change frees it once the reader has let go of it",
[indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "---\nid: b\n---\n# Beta" })],
function* ({ editorWrites, readerLetsGoOfRead, readerReadsIndex, releasedReadsInMemory, watcherReports }) {
yield* editorWrites("a.md", "---\nid: a\n---\n# First edit");
yield* watcherReports("a.md");
yield* readerLetsGoOfRead("the first-edit read");
expect(yield* releasedReadsInMemory()).toEqual(["the first-edit read"]);

yield* editorWrites("a.md", "---\nid: a\n---\n# Second edit");
yield* watcherReports("a.md");
yield* readerReadsIndex();

expect(yield* releasedReadsInMemory()).toEqual([]);
},
);
5 changes: 3 additions & 2 deletions packages/work-board/src/server/search/snapshot.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Clock, Effect, type FileSystem, Option, type Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect";
import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, Ref, Semaphore, SubscriptionRef } from "effect";
import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts";
import type { MetadataDocument } from "#integration/metadata.ts";
import type { Indexing } from "#rpc/Indexing.ts";
Expand Down Expand Up @@ -122,7 +122,8 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* (
const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file]));
const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled);
const enriched = enrich(next, model);
const services = yield* Effect.context<FileSystem.FileSystem | Path.Path>();
// Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot.
const services = (yield* Effect.context<FileSystem.FileSystem | Path.Path>()).pipe(Context.pick(FileSystem.FileSystem, Path.Path));
const snapshot: Snapshot = {
archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)),
documents,
Expand Down
13 changes: 13 additions & 0 deletions packages/work-board/src/test-support/collectGarbage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { setFlagsFromString } from "node:v8";
import { runInNewContext } from "node:vm";
import { Effect } from "effect";

let gc: (() => void) | undefined;

export const collectGarbage = Effect.sync(() => {
if (gc === undefined) {
setFlagsFromString("--expose-gc");
gc = runInNewContext("gc");
}
gc?.();
});
36 changes: 36 additions & 0 deletions packages/work-board/src/test-support/indexComparison.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { Order } from "effect";
import type { Snapshot } from "#server/search/snapshot.ts";

// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them.
export function comparable(value: unknown): unknown {
if (value instanceof Map) {
return [...value].map(comparable);
}
if (value instanceof Set) {
return [...value].map(comparable);
}
if (Array.isArray(value)) {
return value.map(comparable);
}
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value)
.filter(([, field]) => field !== undefined)
.map(([key, field]) => [key, comparable(field)]),
);
}
return value;
}

export function kept(before: Snapshot, after: Snapshot) {
return after.documents
.filter((document) => {
const previous = before.documents.find((candidate) => candidate.file === document.file);
function entries(snapshot: Snapshot) {
return snapshot.entries.filter((entry) => entry.file === document.file);
}
return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry);
})
.map((document) => document.file)
.sort(Order.String);
}
52 changes: 17 additions & 35 deletions packages/work-board/src/test-support/indexStory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import { inboxItems } from "#server/attention/inbox.ts";
import { searchMatch } from "#server/search/match.ts";
import { type Snapshot, searchSnapshot } from "#server/search/snapshot.ts";
import { type Folder, folder } from "./board.ts";
import { collectGarbage } from "./collectGarbage.ts";
import { incrementalIndex } from "./incrementalIndex.ts";
import { comparable, kept } from "./indexComparison.ts";
import { indexState } from "./indexState.ts";

interface IndexEngine {
Expand All @@ -19,6 +21,7 @@ interface IndexEngine {
lastRead?: Snapshot;
notes: Folder;
read?: Effect.Success<ReturnType<typeof searchSnapshot>>;
readonly released: Map<string, WeakRef<Snapshot>>;
unreadable?: string;
}

Expand All @@ -41,40 +44,6 @@ function advancingListing(engine: IndexEngine, index: Effect.Success<ReturnType<
return { ...index.changes, inventory: index.changes.inventory.pipe(Effect.tap(() => addFile)) };
}

// Maps and Sets compare by content and missing fields equal undefined ones, as a reader of the index sees them.
function comparable(value: unknown): unknown {
if (value instanceof Map) {
return [...value].map(comparable);
}
if (value instanceof Set) {
return [...value].map(comparable);
}
if (Array.isArray(value)) {
return value.map(comparable);
}
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value)
.filter(([, field]) => field !== undefined)
.map(([key, field]) => [key, comparable(field)]),
);
}
return value;
}

function kept(before: Snapshot, after: Snapshot) {
return after.documents
.filter((document) => {
const previous = before.documents.find((candidate) => candidate.file === document.file);
function entries(snapshot: Snapshot) {
return snapshot.entries.filter((entry) => entry.file === document.file);
}
return previous === document && entries(before).every((entry, index) => entries(after)[index] === entry);
})
.map((document) => document.file)
.sort(Order.String);
}

export const indexStory = storyKit({
after: {
*project(engine) {
Expand All @@ -100,7 +69,7 @@ export const indexStory = storyKit({
Effect.sync(() => folder({})),
(acquired) => Effect.sync(acquired.remove),
);
const engine: IndexEngine = { addDuringListing: false, notes };
const engine: IndexEngine = { addDuringListing: false, notes, released: new Map() };
return engine;
},
inspect: (engine) => ({ blocked: [...(engine.index?.blocked ?? [])], reads: engine.index?.reads, root: engine.notes.root }),
Expand Down Expand Up @@ -227,6 +196,13 @@ export const indexStory = storyKit({
story.tell("the reader counts the path and directory checks of the read");
return indexed(engine).index.checks();
}),
readerLetsGoOfRead: (name: string) => {
// Told before the read: the story's call sites would otherwise keep the read in memory.
story.tell(`the reader reads the index as ${name} and lets go of that read`);
return Effect.map(current(), (snapshot) => {
engine.released.set(name, new WeakRef(snapshot));
});
},
readerReadsIndex: () =>
Effect.map(current(), (snapshot) => {
story.tell("the reader reads the index");
Expand All @@ -243,6 +219,12 @@ export const indexStory = storyKit({
indexed(engine).index.blocked.clear();
indexed(engine).index.reads.length = 0;
}),
releasedReadsInMemory: () =>
Effect.gen(function* () {
story.tell("garbage collection runs and the reader lists the reads it let go of that are still in memory");
yield* collectGarbage;
return [...engine.released].filter(([, read]) => read.deref() !== undefined).map(([name]) => name);
}),
revisionOfRead: () =>
Effect.gen(function* () {
story.tell("the reader compares the read's revision with the index's current revision");
Expand Down
Loading