Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .changeset/quiet-board-rebuilds.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
"@shivaedev/work-board": patch
"@shivaedev/work-fleet": patch
---

### Fixed

- `@shivaedev/work-board`: The Board index keeps the decoded settled catalogue and archive state until those files change, so a reported edit reads only the edited file instead of reading the whole catalogue again on every rebuild.
- `@shivaedev/work-fleet`: A fresh read of a brief whose source did not change no longer invalidates the Board index, so reconciling done work no longer rebuilds the index for each record.
- `@shivaedev/work-fleet`: Fleet renders `fleet/status.md` and counts health only when its records change, and checks each Board snapshot's problems once instead of on every read pass.
27 changes: 23 additions & 4 deletions packages/work-board/src/server/archive/documents.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Effect, FileSystem, Order, Path, Schema } from "effect";
import { Effect, FileSystem, Order, Path, type PlatformError, Schema } from "effect";
import type { MetadataDocument } from "#integration/metadata.ts";
import { cachedRead } from "#server/files/cachedRead.ts";
import { type Changes, refreshWritten } from "#server/files/changes.ts";

const RETENTION = 14 * 24 * 60 * 60 * 1000;
Expand All @@ -23,16 +24,34 @@ function identity(document: MetadataDocument) {
return document.parsed.fields.id ?? document.file.replace(/^(?:tasks|archive)\//u, "");
}

export interface ArchiveState {
readonly restored: ReadonlySet<string>;
readonly saved: string;
}

const STATE_FILE = ".board-archive.json";

function decodeState(saved: string) {
return Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe(
Effect.orDie,
Effect.map((identities): ArchiveState => ({ restored: new Set(identities), saved })),
);
}

export function archiveStateReader(root: string) {
return cachedRead(root, STATE_FILE, { restored: new Set<string>(), saved: "[]" }, decodeState);
}

export const archiveDocuments = Effect.fn("WorkBoard.archiveDocuments")(function* (
root: string,
documents: readonly MetadataDocument[],
now: number,
state?: Effect.Effect<ArchiveState, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path>,
) {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const stateFile = path.join(root, ".board-archive.json");
const saved = (yield* fs.exists(stateFile)) ? yield* fs.readFileString(stateFile) : "[]";
const restored = new Set(yield* Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe(Effect.orDie));
const stateFile = path.join(root, STATE_FILE);
const { restored, saved } = yield* state ?? archiveStateReader(root);
const archived = new Set(documents.filter((document) => document.file.startsWith("archive/")).map(identity));
const moved: string[] = [];
for (const document of documents) {
Expand Down
13 changes: 9 additions & 4 deletions packages/work-board/src/server/archive/loadSettled.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
import { Effect, FileSystem, Path } from "effect";
import { Effect, FileSystem, Path, type PlatformError } from "effect";
import { InternalServerError } from "@shivaedev/errors/taxonomy.ts";
import { documentSource, metadataModel } from "#server/metadata/model.ts";
import { readDocument } from "#server/search/readDocument.ts";
import type { Snapshot } from "#server/search/snapshot.ts";
import { readCatalogue, recordPath } from "./settledCatalogue.ts";
import { catalogueReader, readCatalogue, recordPath, type SettledCatalogue } from "./settledCatalogue.ts";

export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* (root: string, snapshot: Snapshot, selection: string) {
const catalogue = yield* readCatalogue(root);
export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* (
root: string,
snapshot: Snapshot,
selection: string,
settled?: Effect.Effect<SettledCatalogue, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path>,
) {
const { records: catalogue } = yield* settled ?? catalogueReader(root);
const groups = new Set(catalogue.filter((record) => record.id === selection || record.file === selection).map((record) => record.item));
groups.add(selection);
const selected = snapshot.documents.find((document) => document.file === selection)?.parsed.fields.id;
Expand Down
17 changes: 10 additions & 7 deletions packages/work-board/src/server/archive/settled.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Effect, FileSystem, Path } from "effect";
import { Effect, FileSystem, Path, type PlatformError } from "effect";
import type { MetadataDocument } from "#integration/metadata.ts";
import { documentSource } from "#server/metadata/model.ts";
import { appendCatalogue, readCatalogue, recordPath, type SettledRecord } from "./settledCatalogue.ts";
import { appendCatalogue, catalogueReader, recordPath, type SettledCatalogue, type SettledRecord } from "./settledCatalogue.ts";
import { settledPlan } from "./settledPlan.ts";
import { settledRecords } from "./settledRecords.ts";

Expand Down Expand Up @@ -37,7 +37,11 @@ const moveSettled = Effect.fn("WorkBoard.moveSettled")(
),
);

export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (root: string, documents: readonly MetadataDocument[]) {
export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (
root: string,
documents: readonly MetadataDocument[],
settled?: Effect.Effect<SettledCatalogue, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path>,
) {
const ids = new Set(documents.map((document) => document.parsed.fields.id));
const pending = documents.some(
({ parsed: { fields } }) =>
Expand All @@ -48,10 +52,10 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r
if (!pending && settledRecords(documents).length === 0) {
return [];
}
const catalogue = yield* readCatalogue(root);
const catalogue = yield* settled ?? catalogueReader(root);
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const planned = yield* Effect.filter(settledPlan(documents, catalogue), ({ document }) =>
const planned = yield* Effect.filter(settledPlan(documents, catalogue.byId), ({ document }) =>
fs.readFileString(path.join(root, document.file)).pipe(
Effect.map((source) => source === documentSource(document)),
Effect.catchIf(
Expand All @@ -63,8 +67,7 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r
if (planned.length === 0) {
return [];
}
const existing = new Set(catalogue.map((record) => record.id));
const added = planned.map(({ record }) => record).filter((record) => !existing.has(record.id));
const added = planned.map(({ record }) => record).filter((record) => !catalogue.byId.has(record.id));
// Publish locations before removing sources, so an interrupted move remains discoverable and can finish on the next read.
yield* appendCatalogue(root, added);
return (yield* Effect.forEach(planned, ({ document, record }) => moveSettled(root, document, record))).flat();
Expand Down
29 changes: 24 additions & 5 deletions packages/work-board/src/server/archive/settledCatalogue.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Effect, FileSystem, Path, Schema } from "effect";
import { cachedRead } from "#server/files/cachedRead.ts";

const Record = Schema.Struct({
file: Schema.String,
Expand All @@ -11,22 +12,40 @@ const Record = Schema.Struct({
const CatalogueEntry = Schema.fromJsonString(Record);
export type SettledRecord = typeof Record.Type;

export interface SettledCatalogue {
readonly byId: ReadonlyMap<string, SettledRecord>;
readonly files: ReadonlyMap<string, string>;
readonly records: readonly SettledRecord[];
}

const CATALOGUE = ".board-settled/catalogue.jsonl";
const EMPTY: SettledCatalogue = { byId: new Map(), files: new Map(), records: [] };

const decodeCatalogue = Effect.fn("WorkBoard.decodeCatalogue")(function* (source: string) {
const lines = source.split("\n").filter((line) => line.length > 0);
const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie));
const byId = new Map(records.map((record) => [record.id, record]));
return { byId, files: new Map([...byId].map(([id, record]) => [id, record.file])), records: [...byId.values()] } satisfies SettledCatalogue;
});

export const readCatalogue = Effect.fn("WorkBoard.readCatalogue")(function* (root: string) {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const file = path.join(root, ".board-settled/catalogue.jsonl");
const file = path.join(root, CATALOGUE);
if (!(yield* fs.exists(file))) {
return [];
}
const lines = (yield* fs.readFileString(file)).split("\n").filter((line) => line.length > 0);
const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie));
return [...new Map(records.map((record) => [record.id, record])).values()];
return (yield* decodeCatalogue(yield* fs.readFileString(file))).records;
});

export function catalogueReader(root: string) {
return cachedRead(root, CATALOGUE, EMPTY, decodeCatalogue);
}

export const appendCatalogue = Effect.fn("WorkBoard.appendCatalogue")(function* (root: string, records: readonly SettledRecord[]) {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const file = path.join(root, ".board-settled/catalogue.jsonl");
const file = path.join(root, CATALOGUE);
yield* fs.makeDirectory(path.dirname(file), { recursive: true });
for (const record of records) {
yield* fs.writeFileString(file, `${JSON.stringify(record)}\n`, { flag: "a" });
Expand Down
3 changes: 1 addition & 2 deletions packages/work-board/src/server/archive/settledPlan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import { revisionOf } from "#server/responses/records.ts";
import type { SettledRecord } from "./settledCatalogue.ts";
import { settledRecords } from "./settledRecords.ts";

export function settledPlan(documents: readonly MetadataDocument[], catalogue: readonly SettledRecord[]) {
const existing = new Map(catalogue.map((record) => [record.id, record]));
export function settledPlan(documents: readonly MetadataDocument[], existing: ReadonlyMap<string, SettledRecord>) {
const candidates = new Map(settledRecords(documents).map((candidate) => [candidate.id, candidate]));
return documents.flatMap((document) => {
const id = document.parsed.fields.id ?? "";
Expand Down
32 changes: 32 additions & 0 deletions packages/work-board/src/server/files/cachedRead.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { Effect, FileSystem, Option, Path } from "effect";

function version(info: FileSystem.File.Info) {
return [info.dev, Option.getOrUndefined(info.ino), info.size, Option.getOrUndefined(Option.map(info.mtime, (date) => date.getTime()))].join(":");
}

// Every rebuild consults these bookkeeping files, so their decoded value is kept until the file's identity, size or modification time moves.
export function cachedRead<A, E, R>(root: string, name: string, missing: A, decode: (source: string) => Effect.Effect<A, E, R>) {
let cached: { readonly value: A; readonly version: string } | undefined;
return Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const file = (yield* Path.Path).join(root, name);
const info = yield* fs.stat(file).pipe(
Effect.map(Option.some),
Effect.catchIf(
(error) => error.reason._tag === "NotFound",
() => Effect.succeed(Option.none<FileSystem.File.Info>()),
),
);
if (Option.isNone(info)) {
cached = undefined;
return missing;
}
const current = version(info.value);
if (cached?.version === current) {
return cached.value;
}
const value = yield* decode(yield* fs.readFileString(file));
cached = { value, version: current };
return value;
});
}
3 changes: 3 additions & 0 deletions packages/work-board/src/server/metadata/updateModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import { metadataProblems } from "./problems.ts";
import { referencesOf } from "./referencesOf.ts";

function changedSettledIds(before: ReadonlyMap<string, string> | undefined, after: ReadonlyMap<string, string> | undefined) {
if (before === after) {
return [];
}
return [...new Set([...(before?.keys() ?? []), ...(after?.keys() ?? [])])].filter((id) => before?.get(id) !== after?.get(id));
}

Expand Down
14 changes: 11 additions & 3 deletions packages/work-board/src/server/search/archiveInventory.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
import { Effect } from "effect";
import { Effect, type FileSystem, type Path, type PlatformError } from "effect";
import type { MetadataDocument } from "#integration/metadata.ts";
import { archiveDocuments } from "#server/archive/documents.ts";
import { type ArchiveState, archiveDocuments } from "#server/archive/documents.ts";
import { archiveSettled } from "#server/archive/settled.ts";
import type { SettledCatalogue } from "#server/archive/settledCatalogue.ts";

export interface ArchiveReads {
readonly catalogue: Effect.Effect<SettledCatalogue, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path>;
readonly state: Effect.Effect<ArchiveState, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path>;
}

export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function* (
root: string,
documents: readonly MetadataDocument[],
modified: ReadonlyMap<string, number>,
now: number,
unavailable: readonly string[],
reads: ArchiveReads,
) {
if (unavailable.length > 0) {
return [];
Expand All @@ -17,6 +24,7 @@ export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function
root,
documents.map((document) => ({ ...document, modified: modified.get(document.file) ?? now })),
now,
reads.state,
);
return [...moved, ...(yield* archiveSettled(root, documents))];
return [...moved, ...(yield* archiveSettled(root, documents, reads.catalogue))];
});
27 changes: 27 additions & 0 deletions packages/work-board/src/server/search/readCosts.spec.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { expect } from "vitest";
import { outcomes } from "@shivaedev/test-story/server/outcomes.ts";
import { indexedAliases } from "#test/indexedAliases.ts";
import { indexedSettled } from "#test/indexedSettled.ts";
import { indexedFiles, indexStory } from "#test/indexStory.ts";

const NOTES = 100;
Expand Down Expand Up @@ -30,3 +31,29 @@ indexStory.it(
]);
},
);

indexStory.it(
"a reported edit reads only the edited file while the settled catalogue and archive state stand unchanged",
[indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "# Beta" }), indexedSettled(["settled-one", "settled-two"])],
function* ({ editorWrites, filesRead, settledIds, watcherReports }) {
yield* editorWrites("a.md", "---\nid: a\n---\n# Alpha, revised");
yield* watcherReports("a.md");

yield* outcomes([
["files read", () => filesRead(), ["a.md"]],
["settled records", () => settledIds(), ["settled-one", "settled-two"]],
]);
},
);

indexStory.it(
"the next reported change picks up a record another process appended to the settled catalogue",
[indexedFiles({ "a.md": "# Alpha" }), indexedSettled(["settled-one"])],
function* ({ anotherProcessSettles, editorWrites, settledIds, watcherReports }) {
yield* anotherProcessSettles("settled-two");
yield* editorWrites("a.md", "# Alpha, revised");
yield* watcherReports("a.md");

expect(yield* settledIds()).toEqual(["settled-one", "settled-two"]);
},
);
13 changes: 7 additions & 6 deletions packages/work-board/src/server/search/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, R
import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts";
import type { MetadataDocument } from "#integration/metadata.ts";
import type { Indexing } from "#rpc/Indexing.ts";
import { refreshArchived } from "#server/archive/documents.ts";
import { archiveStateReader, refreshArchived } from "#server/archive/documents.ts";
import { loadSettled } from "#server/archive/loadSettled.ts";
import { readCatalogue } from "#server/archive/settledCatalogue.ts";
import { catalogueReader } from "#server/archive/settledCatalogue.ts";
import type { Changes } from "#server/files/changes.ts";
import { invalidationsSince } from "#server/files/invalidations.ts";
import type { MarkdownFile } from "#server/files/list.ts";
import { markdownPaths } from "#server/files/markdownPath.ts";
import type { metadataModel } from "#server/metadata/model.ts";
import { updateModel } from "#server/metadata/updateModel.ts";
import { archiveInventory } from "./archiveInventory.ts";
import { type ArchiveReads, archiveInventory } from "./archiveInventory.ts";
import type { Entry } from "./entries.ts";
import { fileCounter } from "./fileCounter.ts";
import { type IndexedDocument, readDocument } from "./readDocument.ts";
Expand Down Expand Up @@ -73,6 +73,7 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* (
let records = new Map<string, IndexedDocument>();
let listed = new Map<string, MarkdownFile>();
const enrich = updateEntries();
const archiveReads: ArchiveReads = { catalogue: catalogueReader(root), state: archiveStateReader(root) };
function find(file: MarkdownFile, dirty: ReadonlySet<string> | undefined, resolve: (file: string) => Effect.Effect<string | undefined>) {
const previous = records.get(file.path);
return shouldRead(file, previous, listed, dirty)
Expand Down Expand Up @@ -113,19 +114,19 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* (
}
}
const modified = new Map(files.map((file) => [file.path, file.modified]));
const moved = yield* archiveInventory(root, documents, modified, now, unavailable);
const moved = yield* archiveInventory(root, documents, modified, now, unavailable, archiveReads);
housekeptAt = now;
if (moved.length > 0) {
yield* refreshArchived(changes, moved);
return yield* buildSnapshot(hit);
}
const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file]));
const { files: settled } = yield* archiveReads.catalogue;
const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled);
const enriched = enrich(next, model);
// Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot.
const services = (yield* Effect.context<FileSystem.FileSystem | Path.Path>()).pipe(Context.pick(FileSystem.FileSystem, Path.Path));
const snapshot: Snapshot = {
archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)),
archived: (selection) => loadSettled(root, snapshot, selection, archiveReads.catalogue).pipe(Effect.provideContext(services)),
documents,
entries: enriched,
maintainer,
Expand Down
3 changes: 2 additions & 1 deletion packages/work-board/src/server/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ export const Workspace = defineService({
}
return {
...metadata,
archived: (selection: string) => Effect.flatMap(index, (snapshot) => loadSettled(options.root, snapshot, selection)),
archived: (selection: string) =>
Effect.flatMap(index, (snapshot) => snapshot.archived?.(selection) ?? loadSettled(options.root, snapshot, selection)),
document,
index: () => index,
settings: () => Effect.succeed(settings),
Expand Down
13 changes: 12 additions & 1 deletion packages/work-board/src/test-support/indexStory.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs";
import { appendFileSync, existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs";
import { join, relative } from "node:path";
import { isDeepStrictEqual } from "node:util";
import { NodeServices } from "@effect/platform-node";
Expand All @@ -12,6 +12,7 @@ import { collectGarbage } from "./collectGarbage.ts";
import { incrementalIndex } from "./incrementalIndex.ts";
import { comparable, kept } from "./indexComparison.ts";
import { indexState } from "./indexState.ts";
import { SETTLED_CATALOGUE, settledLine } from "./settledCatalogue.ts";

interface IndexEngine {
addDuringListing: boolean;
Expand Down Expand Up @@ -87,6 +88,11 @@ export const indexStory = storyKit({
return engine.before;
}
return {
anotherProcessSettles: (id: string) =>
Effect.sync(() => {
story.tell(`another process records ${id} in the settled catalogue`);
appendFileSync(join(engine.notes.root, SETTLED_CATALOGUE), settledLine(id));
}),
bodyOf: (file: string) =>
Effect.map(current(), (snapshot) => {
story.tell(`the reader reads the indexed body of ${file}`);
Expand Down Expand Up @@ -241,6 +247,11 @@ export const indexStory = storyKit({
story.tell(`the reader searches the index for ${JSON.stringify(query)} and reads the title of ${id}`);
return searchMatch(snapshot.entries, query).results.find((entry) => entry.id === id)?.title ?? "not found";
}),
settledIds: () =>
Effect.map(current(), (snapshot) => {
story.tell("the reader reads which settled records the index knows");
return [...(snapshot.model.settled?.keys() ?? [])].sort(Order.String);
}),
unavailableFiles: () =>
Effect.map(current(), (snapshot) => {
story.tell("the reader reads which files the index could not read");
Expand Down
Loading
Loading