From c13b6f1146e4177fd7f5f4e746300162c3cb9450 Mon Sep 17 00:00:00 2001 From: Marvin Bitterlich <644950+marvin-bitterlich@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:58:21 +0000 Subject: [PATCH] Fleet and Board stop redoing whole-board work for unchanged files A fresh read of an unchanged brief no longer invalidates the Board index, the index keeps the decoded settled catalogue and archive state until those files change, Fleet renders status and counts health only when its records change, and Fleet checks a snapshot's problems once. --- .changeset/quiet-board-rebuilds.md | 10 +++++ .../src/server/archive/documents.ts | 27 ++++++++++-- .../src/server/archive/loadSettled.ts | 13 ++++-- .../work-board/src/server/archive/settled.ts | 17 ++++---- .../src/server/archive/settledCatalogue.ts | 29 ++++++++++--- .../src/server/archive/settledPlan.ts | 3 +- .../work-board/src/server/files/cachedRead.ts | 32 ++++++++++++++ .../src/server/metadata/updateModel.ts | 3 ++ .../src/server/search/archiveInventory.ts | 14 +++++-- .../src/server/search/readCosts.spec.ts | 27 ++++++++++++ .../work-board/src/server/search/snapshot.ts | 13 +++--- packages/work-board/src/server/workspace.ts | 3 +- .../work-board/src/test-support/indexStory.ts | 13 +++++- .../src/test-support/indexedSettled.ts | 9 ++++ .../src/test-support/servingBoardVerbs.ts | 6 ++- .../src/test-support/settledCatalogue.ts | 5 +++ packages/work-fleet/quality-baseline.jsonl | 2 +- .../work-fleet/src/board/freshRead.spec.ts | 25 ++++++++++- packages/work-fleet/src/board/readWork.ts | 42 +++++++++++-------- .../src/board/warnSnapshotProblems.ts | 6 +++ .../work-fleet/src/host/publishedRecords.ts | 41 ++++++++++++++++++ .../work-fleet/src/host/statusLayer.test.ts | 15 +++++++ packages/work-fleet/src/host/statusLayer.ts | 27 ++++++------ .../src/test-support/boardReadStory.ts | 39 ++++++++++++++++- .../work-fleet/src/test-support/boardReads.ts | 11 +++-- 25 files changed, 358 insertions(+), 74 deletions(-) create mode 100644 .changeset/quiet-board-rebuilds.md create mode 100644 packages/work-board/src/server/files/cachedRead.ts create mode 100644 packages/work-board/src/test-support/indexedSettled.ts create mode 100644 packages/work-board/src/test-support/settledCatalogue.ts create mode 100644 packages/work-fleet/src/host/publishedRecords.ts diff --git a/.changeset/quiet-board-rebuilds.md b/.changeset/quiet-board-rebuilds.md new file mode 100644 index 00000000..f91ccbf0 --- /dev/null +++ b/.changeset/quiet-board-rebuilds.md @@ -0,0 +1,10 @@ +--- +"@shivaedev/work-board": patch +"@shivaedev/work-fleet": patch +--- + +### Fixed + +- `@shivaedev/work-board`: The Board index keeps the decoded settled catalogue and archive state until those files change, so a reported edit reads only the edited file instead of reading the whole catalogue again on every rebuild. +- `@shivaedev/work-fleet`: A fresh read of a brief whose source did not change no longer invalidates the Board index, so reconciling done work no longer rebuilds the index for each record. +- `@shivaedev/work-fleet`: Fleet renders `fleet/status.md` and counts health only when its records change, and checks each Board snapshot's problems once instead of on every read pass. diff --git a/packages/work-board/src/server/archive/documents.ts b/packages/work-board/src/server/archive/documents.ts index c57ddb09..57060303 100644 --- a/packages/work-board/src/server/archive/documents.ts +++ b/packages/work-board/src/server/archive/documents.ts @@ -1,5 +1,6 @@ -import { Effect, FileSystem, Order, Path, Schema } from "effect"; +import { Effect, FileSystem, Order, Path, type PlatformError, Schema } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; +import { cachedRead } from "#server/files/cachedRead.ts"; import { type Changes, refreshWritten } from "#server/files/changes.ts"; const RETENTION = 14 * 24 * 60 * 60 * 1000; @@ -23,16 +24,34 @@ function identity(document: MetadataDocument) { return document.parsed.fields.id ?? document.file.replace(/^(?:tasks|archive)\//u, ""); } +export interface ArchiveState { + readonly restored: ReadonlySet; + readonly saved: string; +} + +const STATE_FILE = ".board-archive.json"; + +function decodeState(saved: string) { + return Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe( + Effect.orDie, + Effect.map((identities): ArchiveState => ({ restored: new Set(identities), saved })), + ); +} + +export function archiveStateReader(root: string) { + return cachedRead(root, STATE_FILE, { restored: new Set(), saved: "[]" }, decodeState); +} + export const archiveDocuments = Effect.fn("WorkBoard.archiveDocuments")(function* ( root: string, documents: readonly MetadataDocument[], now: number, + state?: Effect.Effect, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const stateFile = path.join(root, ".board-archive.json"); - const saved = (yield* fs.exists(stateFile)) ? yield* fs.readFileString(stateFile) : "[]"; - const restored = new Set(yield* Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe(Effect.orDie)); + const stateFile = path.join(root, STATE_FILE); + const { restored, saved } = yield* state ?? archiveStateReader(root); const archived = new Set(documents.filter((document) => document.file.startsWith("archive/")).map(identity)); const moved: string[] = []; for (const document of documents) { diff --git a/packages/work-board/src/server/archive/loadSettled.ts b/packages/work-board/src/server/archive/loadSettled.ts index 93cc6dd4..0d3531bf 100644 --- a/packages/work-board/src/server/archive/loadSettled.ts +++ b/packages/work-board/src/server/archive/loadSettled.ts @@ -1,12 +1,17 @@ -import { Effect, FileSystem, Path } from "effect"; +import { Effect, FileSystem, Path, type PlatformError } from "effect"; import { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import { documentSource, metadataModel } from "#server/metadata/model.ts"; import { readDocument } from "#server/search/readDocument.ts"; import type { Snapshot } from "#server/search/snapshot.ts"; -import { readCatalogue, recordPath } from "./settledCatalogue.ts"; +import { catalogueReader, readCatalogue, recordPath, type SettledCatalogue } from "./settledCatalogue.ts"; -export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* (root: string, snapshot: Snapshot, selection: string) { - const catalogue = yield* readCatalogue(root); +export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* ( + root: string, + snapshot: Snapshot, + selection: string, + settled?: Effect.Effect, +) { + const { records: catalogue } = yield* settled ?? catalogueReader(root); const groups = new Set(catalogue.filter((record) => record.id === selection || record.file === selection).map((record) => record.item)); groups.add(selection); const selected = snapshot.documents.find((document) => document.file === selection)?.parsed.fields.id; diff --git a/packages/work-board/src/server/archive/settled.ts b/packages/work-board/src/server/archive/settled.ts index 4e6e46f9..1eb60ce0 100644 --- a/packages/work-board/src/server/archive/settled.ts +++ b/packages/work-board/src/server/archive/settled.ts @@ -1,7 +1,7 @@ -import { Effect, FileSystem, Path } from "effect"; +import { Effect, FileSystem, Path, type PlatformError } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; import { documentSource } from "#server/metadata/model.ts"; -import { appendCatalogue, readCatalogue, recordPath, type SettledRecord } from "./settledCatalogue.ts"; +import { appendCatalogue, catalogueReader, recordPath, type SettledCatalogue, type SettledRecord } from "./settledCatalogue.ts"; import { settledPlan } from "./settledPlan.ts"; import { settledRecords } from "./settledRecords.ts"; @@ -37,7 +37,11 @@ const moveSettled = Effect.fn("WorkBoard.moveSettled")( ), ); -export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (root: string, documents: readonly MetadataDocument[]) { +export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* ( + root: string, + documents: readonly MetadataDocument[], + settled?: Effect.Effect, +) { const ids = new Set(documents.map((document) => document.parsed.fields.id)); const pending = documents.some( ({ parsed: { fields } }) => @@ -48,10 +52,10 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r if (!pending && settledRecords(documents).length === 0) { return []; } - const catalogue = yield* readCatalogue(root); + const catalogue = yield* settled ?? catalogueReader(root); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const planned = yield* Effect.filter(settledPlan(documents, catalogue), ({ document }) => + const planned = yield* Effect.filter(settledPlan(documents, catalogue.byId), ({ document }) => fs.readFileString(path.join(root, document.file)).pipe( Effect.map((source) => source === documentSource(document)), Effect.catchIf( @@ -63,8 +67,7 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r if (planned.length === 0) { return []; } - const existing = new Set(catalogue.map((record) => record.id)); - const added = planned.map(({ record }) => record).filter((record) => !existing.has(record.id)); + const added = planned.map(({ record }) => record).filter((record) => !catalogue.byId.has(record.id)); // Publish locations before removing sources, so an interrupted move remains discoverable and can finish on the next read. yield* appendCatalogue(root, added); return (yield* Effect.forEach(planned, ({ document, record }) => moveSettled(root, document, record))).flat(); diff --git a/packages/work-board/src/server/archive/settledCatalogue.ts b/packages/work-board/src/server/archive/settledCatalogue.ts index f190260d..21014ee0 100644 --- a/packages/work-board/src/server/archive/settledCatalogue.ts +++ b/packages/work-board/src/server/archive/settledCatalogue.ts @@ -1,4 +1,5 @@ import { Effect, FileSystem, Path, Schema } from "effect"; +import { cachedRead } from "#server/files/cachedRead.ts"; const Record = Schema.Struct({ file: Schema.String, @@ -11,22 +12,40 @@ const Record = Schema.Struct({ const CatalogueEntry = Schema.fromJsonString(Record); export type SettledRecord = typeof Record.Type; +export interface SettledCatalogue { + readonly byId: ReadonlyMap; + readonly files: ReadonlyMap; + readonly records: readonly SettledRecord[]; +} + +const CATALOGUE = ".board-settled/catalogue.jsonl"; +const EMPTY: SettledCatalogue = { byId: new Map(), files: new Map(), records: [] }; + +const decodeCatalogue = Effect.fn("WorkBoard.decodeCatalogue")(function* (source: string) { + const lines = source.split("\n").filter((line) => line.length > 0); + const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie)); + const byId = new Map(records.map((record) => [record.id, record])); + return { byId, files: new Map([...byId].map(([id, record]) => [id, record.file])), records: [...byId.values()] } satisfies SettledCatalogue; +}); + export const readCatalogue = Effect.fn("WorkBoard.readCatalogue")(function* (root: string) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const file = path.join(root, ".board-settled/catalogue.jsonl"); + const file = path.join(root, CATALOGUE); if (!(yield* fs.exists(file))) { return []; } - const lines = (yield* fs.readFileString(file)).split("\n").filter((line) => line.length > 0); - const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie)); - return [...new Map(records.map((record) => [record.id, record])).values()]; + return (yield* decodeCatalogue(yield* fs.readFileString(file))).records; }); +export function catalogueReader(root: string) { + return cachedRead(root, CATALOGUE, EMPTY, decodeCatalogue); +} + export const appendCatalogue = Effect.fn("WorkBoard.appendCatalogue")(function* (root: string, records: readonly SettledRecord[]) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const file = path.join(root, ".board-settled/catalogue.jsonl"); + const file = path.join(root, CATALOGUE); yield* fs.makeDirectory(path.dirname(file), { recursive: true }); for (const record of records) { yield* fs.writeFileString(file, `${JSON.stringify(record)}\n`, { flag: "a" }); diff --git a/packages/work-board/src/server/archive/settledPlan.ts b/packages/work-board/src/server/archive/settledPlan.ts index d66b1618..a8d36902 100644 --- a/packages/work-board/src/server/archive/settledPlan.ts +++ b/packages/work-board/src/server/archive/settledPlan.ts @@ -4,8 +4,7 @@ import { revisionOf } from "#server/responses/records.ts"; import type { SettledRecord } from "./settledCatalogue.ts"; import { settledRecords } from "./settledRecords.ts"; -export function settledPlan(documents: readonly MetadataDocument[], catalogue: readonly SettledRecord[]) { - const existing = new Map(catalogue.map((record) => [record.id, record])); +export function settledPlan(documents: readonly MetadataDocument[], existing: ReadonlyMap) { const candidates = new Map(settledRecords(documents).map((candidate) => [candidate.id, candidate])); return documents.flatMap((document) => { const id = document.parsed.fields.id ?? ""; diff --git a/packages/work-board/src/server/files/cachedRead.ts b/packages/work-board/src/server/files/cachedRead.ts new file mode 100644 index 00000000..efa866b9 --- /dev/null +++ b/packages/work-board/src/server/files/cachedRead.ts @@ -0,0 +1,32 @@ +import { Effect, FileSystem, Option, Path } from "effect"; + +function version(info: FileSystem.File.Info) { + return [info.dev, Option.getOrUndefined(info.ino), info.size, Option.getOrUndefined(Option.map(info.mtime, (date) => date.getTime()))].join(":"); +} + +// Every rebuild consults these bookkeeping files, so their decoded value is kept until the file's identity, size or modification time moves. +export function cachedRead(root: string, name: string, missing: A, decode: (source: string) => Effect.Effect) { + let cached: { readonly value: A; readonly version: string } | undefined; + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const file = (yield* Path.Path).join(root, name); + const info = yield* fs.stat(file).pipe( + Effect.map(Option.some), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + ); + if (Option.isNone(info)) { + cached = undefined; + return missing; + } + const current = version(info.value); + if (cached?.version === current) { + return cached.value; + } + const value = yield* decode(yield* fs.readFileString(file)); + cached = { value, version: current }; + return value; + }); +} diff --git a/packages/work-board/src/server/metadata/updateModel.ts b/packages/work-board/src/server/metadata/updateModel.ts index f42d8449..5489e4fe 100644 --- a/packages/work-board/src/server/metadata/updateModel.ts +++ b/packages/work-board/src/server/metadata/updateModel.ts @@ -3,6 +3,9 @@ import { metadataProblems } from "./problems.ts"; import { referencesOf } from "./referencesOf.ts"; function changedSettledIds(before: ReadonlyMap | undefined, after: ReadonlyMap | undefined) { + if (before === after) { + return []; + } return [...new Set([...(before?.keys() ?? []), ...(after?.keys() ?? [])])].filter((id) => before?.get(id) !== after?.get(id)); } diff --git a/packages/work-board/src/server/search/archiveInventory.ts b/packages/work-board/src/server/search/archiveInventory.ts index 27cc213e..368e5a81 100644 --- a/packages/work-board/src/server/search/archiveInventory.ts +++ b/packages/work-board/src/server/search/archiveInventory.ts @@ -1,7 +1,13 @@ -import { Effect } from "effect"; +import { Effect, type FileSystem, type Path, type PlatformError } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; -import { archiveDocuments } from "#server/archive/documents.ts"; +import { type ArchiveState, archiveDocuments } from "#server/archive/documents.ts"; import { archiveSettled } from "#server/archive/settled.ts"; +import type { SettledCatalogue } from "#server/archive/settledCatalogue.ts"; + +export interface ArchiveReads { + readonly catalogue: Effect.Effect; + readonly state: Effect.Effect; +} export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function* ( root: string, @@ -9,6 +15,7 @@ export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function modified: ReadonlyMap, now: number, unavailable: readonly string[], + reads: ArchiveReads, ) { if (unavailable.length > 0) { return []; @@ -17,6 +24,7 @@ export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function root, documents.map((document) => ({ ...document, modified: modified.get(document.file) ?? now })), now, + reads.state, ); - return [...moved, ...(yield* archiveSettled(root, documents))]; + return [...moved, ...(yield* archiveSettled(root, documents, reads.catalogue))]; }); diff --git a/packages/work-board/src/server/search/readCosts.spec.ts b/packages/work-board/src/server/search/readCosts.spec.ts index c4cbb95d..97c501a7 100644 --- a/packages/work-board/src/server/search/readCosts.spec.ts +++ b/packages/work-board/src/server/search/readCosts.spec.ts @@ -1,6 +1,7 @@ import { expect } from "vitest"; import { outcomes } from "@shivaedev/test-story/server/outcomes.ts"; import { indexedAliases } from "#test/indexedAliases.ts"; +import { indexedSettled } from "#test/indexedSettled.ts"; import { indexedFiles, indexStory } from "#test/indexStory.ts"; const NOTES = 100; @@ -30,3 +31,29 @@ indexStory.it( ]); }, ); + +indexStory.it( + "a reported edit reads only the edited file while the settled catalogue and archive state stand unchanged", + [indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "# Beta" }), indexedSettled(["settled-one", "settled-two"])], + function* ({ editorWrites, filesRead, settledIds, watcherReports }) { + yield* editorWrites("a.md", "---\nid: a\n---\n# Alpha, revised"); + yield* watcherReports("a.md"); + + yield* outcomes([ + ["files read", () => filesRead(), ["a.md"]], + ["settled records", () => settledIds(), ["settled-one", "settled-two"]], + ]); + }, +); + +indexStory.it( + "the next reported change picks up a record another process appended to the settled catalogue", + [indexedFiles({ "a.md": "# Alpha" }), indexedSettled(["settled-one"])], + function* ({ anotherProcessSettles, editorWrites, settledIds, watcherReports }) { + yield* anotherProcessSettles("settled-two"); + yield* editorWrites("a.md", "# Alpha, revised"); + yield* watcherReports("a.md"); + + expect(yield* settledIds()).toEqual(["settled-one", "settled-two"]); + }, +); diff --git a/packages/work-board/src/server/search/snapshot.ts b/packages/work-board/src/server/search/snapshot.ts index cae18527..2e4390b4 100644 --- a/packages/work-board/src/server/search/snapshot.ts +++ b/packages/work-board/src/server/search/snapshot.ts @@ -2,16 +2,16 @@ import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, R import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import type { MetadataDocument } from "#integration/metadata.ts"; import type { Indexing } from "#rpc/Indexing.ts"; -import { refreshArchived } from "#server/archive/documents.ts"; +import { archiveStateReader, refreshArchived } from "#server/archive/documents.ts"; import { loadSettled } from "#server/archive/loadSettled.ts"; -import { readCatalogue } from "#server/archive/settledCatalogue.ts"; +import { catalogueReader } from "#server/archive/settledCatalogue.ts"; import type { Changes } from "#server/files/changes.ts"; import { invalidationsSince } from "#server/files/invalidations.ts"; import type { MarkdownFile } from "#server/files/list.ts"; import { markdownPaths } from "#server/files/markdownPath.ts"; import type { metadataModel } from "#server/metadata/model.ts"; import { updateModel } from "#server/metadata/updateModel.ts"; -import { archiveInventory } from "./archiveInventory.ts"; +import { type ArchiveReads, archiveInventory } from "./archiveInventory.ts"; import type { Entry } from "./entries.ts"; import { fileCounter } from "./fileCounter.ts"; import { type IndexedDocument, readDocument } from "./readDocument.ts"; @@ -73,6 +73,7 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( let records = new Map(); let listed = new Map(); const enrich = updateEntries(); + const archiveReads: ArchiveReads = { catalogue: catalogueReader(root), state: archiveStateReader(root) }; function find(file: MarkdownFile, dirty: ReadonlySet | undefined, resolve: (file: string) => Effect.Effect) { const previous = records.get(file.path); return shouldRead(file, previous, listed, dirty) @@ -113,19 +114,19 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( } } const modified = new Map(files.map((file) => [file.path, file.modified])); - const moved = yield* archiveInventory(root, documents, modified, now, unavailable); + const moved = yield* archiveInventory(root, documents, modified, now, unavailable, archiveReads); housekeptAt = now; if (moved.length > 0) { yield* refreshArchived(changes, moved); return yield* buildSnapshot(hit); } - const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file])); + const { files: settled } = yield* archiveReads.catalogue; const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled); const enriched = enrich(next, model); // Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot. const services = (yield* Effect.context()).pipe(Context.pick(FileSystem.FileSystem, Path.Path)); const snapshot: Snapshot = { - archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)), + archived: (selection) => loadSettled(root, snapshot, selection, archiveReads.catalogue).pipe(Effect.provideContext(services)), documents, entries: enriched, maintainer, diff --git a/packages/work-board/src/server/workspace.ts b/packages/work-board/src/server/workspace.ts index 32f18c27..1029175b 100644 --- a/packages/work-board/src/server/workspace.ts +++ b/packages/work-board/src/server/workspace.ts @@ -93,7 +93,8 @@ export const Workspace = defineService({ } return { ...metadata, - archived: (selection: string) => Effect.flatMap(index, (snapshot) => loadSettled(options.root, snapshot, selection)), + archived: (selection: string) => + Effect.flatMap(index, (snapshot) => snapshot.archived?.(selection) ?? loadSettled(options.root, snapshot, selection)), document, index: () => index, settings: () => Effect.succeed(settings), diff --git a/packages/work-board/src/test-support/indexStory.ts b/packages/work-board/src/test-support/indexStory.ts index c9a8d62c..918c5cff 100644 --- a/packages/work-board/src/test-support/indexStory.ts +++ b/packages/work-board/src/test-support/indexStory.ts @@ -1,4 +1,4 @@ -import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs"; +import { appendFileSync, existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs"; import { join, relative } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { NodeServices } from "@effect/platform-node"; @@ -12,6 +12,7 @@ import { collectGarbage } from "./collectGarbage.ts"; import { incrementalIndex } from "./incrementalIndex.ts"; import { comparable, kept } from "./indexComparison.ts"; import { indexState } from "./indexState.ts"; +import { SETTLED_CATALOGUE, settledLine } from "./settledCatalogue.ts"; interface IndexEngine { addDuringListing: boolean; @@ -87,6 +88,11 @@ export const indexStory = storyKit({ return engine.before; } return { + anotherProcessSettles: (id: string) => + Effect.sync(() => { + story.tell(`another process records ${id} in the settled catalogue`); + appendFileSync(join(engine.notes.root, SETTLED_CATALOGUE), settledLine(id)); + }), bodyOf: (file: string) => Effect.map(current(), (snapshot) => { story.tell(`the reader reads the indexed body of ${file}`); @@ -241,6 +247,11 @@ export const indexStory = storyKit({ story.tell(`the reader searches the index for ${JSON.stringify(query)} and reads the title of ${id}`); return searchMatch(snapshot.entries, query).results.find((entry) => entry.id === id)?.title ?? "not found"; }), + settledIds: () => + Effect.map(current(), (snapshot) => { + story.tell("the reader reads which settled records the index knows"); + return [...(snapshot.model.settled?.keys() ?? [])].sort(Order.String); + }), unavailableFiles: () => Effect.map(current(), (snapshot) => { story.tell("the reader reads which files the index could not read"); diff --git a/packages/work-board/src/test-support/indexedSettled.ts b/packages/work-board/src/test-support/indexedSettled.ts new file mode 100644 index 00000000..d011c99c --- /dev/null +++ b/packages/work-board/src/test-support/indexedSettled.ts @@ -0,0 +1,9 @@ +import { indexStory } from "./indexStory.ts"; +import { SETTLED_CATALOGUE, settledLine } from "./settledCatalogue.ts"; + +export function indexedSettled(ids: readonly string[]) { + return indexStory.trait("project", `the board keeps settled records for ${ids.join(", ")} and an archive state`, (engine) => { + engine.notes.write(SETTLED_CATALOGUE, ids.map(settledLine).join("")); + engine.notes.write(".board-archive.json", "[]"); + }); +} diff --git a/packages/work-board/src/test-support/servingBoardVerbs.ts b/packages/work-board/src/test-support/servingBoardVerbs.ts index 13a63dd3..e6d2e642 100644 --- a/packages/work-board/src/test-support/servingBoardVerbs.ts +++ b/packages/work-board/src/test-support/servingBoardVerbs.ts @@ -14,6 +14,8 @@ import type { ServingOptions } from "./servingEngine.ts"; import { servingObservations } from "./servingObservations.ts"; import { type BoardEngine, connected, served } from "./storyEngine.ts"; +const BOARD_OWNED = new Set(["board.sock", ".board-archive.json", ".board-settled/catalogue.jsonl"]); + export function servingBoardVerbs(engine: BoardEngine, tell: (line: string) => void) { const { documentAt, linked, changeSeen, launched, pluginBoard, pluginTools } = servingBoardReads(engine, tell); return { @@ -247,13 +249,13 @@ export function servingBoardVerbs(engine: BoardEngine, tell: (line: string) => v }, visitedPaths: () => Effect.sync(() => { - tell("the reader lists every workspace path the board visited, apart from its own socket"); + tell("the reader lists every workspace path the board visited, apart from its own socket and archive bookkeeping files"); const paths = servingObservations(engine).paths; if (!paths) { throw new Error("Add the directoryReadsRecorded trait before listing visited paths"); } return [...new Set(paths.visited.map((path) => relative(engine.notes.root, path)))] - .filter((path) => path !== "board.sock") + .filter((path) => !BOARD_OWNED.has(path)) .sort(Order.String); }), }; diff --git a/packages/work-board/src/test-support/settledCatalogue.ts b/packages/work-board/src/test-support/settledCatalogue.ts new file mode 100644 index 00000000..b86f8908 --- /dev/null +++ b/packages/work-board/src/test-support/settledCatalogue.ts @@ -0,0 +1,5 @@ +export const SETTLED_CATALOGUE = ".board-settled/catalogue.jsonl"; + +export function settledLine(id: string) { + return `${JSON.stringify({ file: `questions/${id}.md`, id, item: id, modified: 0, revision: "settled", targets: [] })}\n`; +} diff --git a/packages/work-fleet/quality-baseline.jsonl b/packages/work-fleet/quality-baseline.jsonl index d1dd7348..80c1761c 100644 --- a/packages/work-fleet/quality-baseline.jsonl +++ b/packages/work-fleet/quality-baseline.jsonl @@ -152,7 +152,7 @@ {"path":"packages/work-fleet/src/host/reportQuestion.spec.ts","rule":"effect/boundary","count":1} {"path":"packages/work-fleet/src/host/runtime.ts","rule":"biome/plugin","count":2} {"path":"packages/work-fleet/src/host/settledAnswers.spec.ts","rule":"effect/boundary","count":1} -{"path":"packages/work-fleet/src/host/statusLayer.ts","rule":"effect/service-definition-assembly","count":6} +{"path":"packages/work-fleet/src/host/statusLayer.ts","rule":"effect/service-definition-assembly","count":5} {"path":"packages/work-fleet/src/host/supervision.spec.ts","rule":"effect/boundary","count":7} {"path":"packages/work-fleet/src/host/supervisor.ts","rule":"effect/boundary","count":1} {"path":"packages/work-fleet/src/host/supervisorPort.ts","rule":"effect/boundary","count":1} diff --git a/packages/work-fleet/src/board/freshRead.spec.ts b/packages/work-fleet/src/board/freshRead.spec.ts index 2fbb5558..21bfa72a 100644 --- a/packages/work-fleet/src/board/freshRead.spec.ts +++ b/packages/work-fleet/src/board/freshRead.spec.ts @@ -1,7 +1,7 @@ import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { outcomes } from "@shivaedev/test-story/server/outcomes.ts"; -import { boardReadStory, hasBoardItems } from "#test/boardReadStory.ts"; +import { boardReadStory, hasBoardItems, namesMissingArea } from "#test/boardReadStory.ts"; import { watchedBoardReads } from "#test/watchedBoardReads.ts"; boardReadStory.it( @@ -70,3 +70,26 @@ boardReadStory.it( ); }, ); + +boardReadStory.it( + "a fresh read of an unchanged brief keeps the Board index without a rebuild", + [hasBoardItems(2)], + function* ({ laterIndexRead, readIndex, readInPass }) { + yield* readInPass("item.0"); + yield* readIndex(); + + yield* readInPass("item.0", { fresh: true }); + + expect(yield* laterIndexRead()).toBe("the same snapshot"); + }, +); + +boardReadStory.it( + "a later read pass over the same Board index checks no area folders again", + [hasBoardItems(1), namesMissingArea()], + function* ({ directoriesReadDuring, readInPass }) { + yield* readInPass("item.0"); + + expect(yield* directoriesReadDuring(readInPass("item.0"))).toBe(0); + }, +); diff --git a/packages/work-fleet/src/board/readWork.ts b/packages/work-fleet/src/board/readWork.ts index 4bc4b889..2f6190ba 100644 --- a/packages/work-fleet/src/board/readWork.ts +++ b/packages/work-fleet/src/board/readWork.ts @@ -1,4 +1,5 @@ import { Effect, Path, Schema } from "effect"; +import type { MetadataDocument } from "@shivaedev/work-board/integration/metadata.ts"; import { documentSource } from "@shivaedev/work-board/server/metadata/model.ts"; import type { Snapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; import { Workspace } from "@shivaedev/work-board/server/workspace.ts"; @@ -9,6 +10,29 @@ import { warnSnapshotProblems } from "#board/warnSnapshotProblems.ts"; import { metadataProblem, workDocument } from "#board/workPolicy.ts"; import { BoardFailure, Priority } from "#policy.ts"; +const freshDocument = Effect.fn("FleetBoard.freshDocument")(function* (realRoot: string, indexed: MetadataDocument, workId: string) { + const workspace = yield* Workspace; + const fresh = yield* workspace + .metadata(indexed.file) + .pipe( + Effect.mapError( + (error) => + new BoardFailure({ code: "incomplete-observation", message: `Board source could not be read: ${indexed.file} (${error.message})` }), + ), + ); + const problem = metadataProblem(fresh); + if (problem) { + return yield* Effect.fail(problem); + } + if (fresh.parsed.fields.id !== workId) { + return yield* Effect.fail(new BoardFailure({ code: "board-item-missing", message: `Board item no longer exists: ${workId}`, workId })); + } + if (documentSource(fresh) !== documentSource(indexed)) { + yield* recordBoardWrite(realRoot, fresh.file); + } + return fresh; +}); + export const readWork = Effect.fn("FleetBoard.readWork")(function* (current: Snapshot, workId: string, options?: { readonly fresh?: boolean }) { const workspace = yield* Workspace; const path = yield* Path.Path; @@ -24,23 +48,7 @@ export const readWork = Effect.fn("FleetBoard.readWork")(function* (current: Sna } let found = yield* workDocument(root, observed, workId); if (options?.fresh) { - const fresh = yield* workspace - .metadata(found.file) - .pipe( - Effect.mapError( - (error) => - new BoardFailure({ code: "incomplete-observation", message: `Board source could not be read: ${found.file} (${error.message})` }), - ), - ); - const problem = metadataProblem(fresh); - if (problem) { - return yield* Effect.fail(problem); - } - if (fresh.parsed.fields.id !== workId) { - return yield* Effect.fail(new BoardFailure({ code: "board-item-missing", message: `Board item no longer exists: ${workId}`, workId })); - } - found = fresh; - yield* recordBoardWrite(realRoot, fresh.file); + found = yield* freshDocument(realRoot, found, workId); } const deliverable = fleetDeliverable(found.parsed); if (deliverable !== undefined && deliverable !== "report") { diff --git a/packages/work-fleet/src/board/warnSnapshotProblems.ts b/packages/work-fleet/src/board/warnSnapshotProblems.ts index ee84db5d..3e052cf8 100644 --- a/packages/work-fleet/src/board/warnSnapshotProblems.ts +++ b/packages/work-fleet/src/board/warnSnapshotProblems.ts @@ -7,8 +7,13 @@ import { metadataProblem, snapshotProblems } from "#board/workPolicy.ts"; import { BoardFailure } from "#policy.ts"; const previous = new Map>(); +// A snapshot never changes, and every read pass consults it, so its problems are checked once; an area folder created outside the Board shows on the next snapshot. +const checked = new WeakSet(); export const warnSnapshotProblems = Effect.fn("FleetBoard.warnSnapshotProblems")(function* (root: string, snapshot: BoardSnapshot) { + if (checked.has(snapshot)) { + return; + } const seen = previous.get(root); const next = new Map(); const problems = snapshotProblems(snapshot); @@ -38,4 +43,5 @@ export const warnSnapshotProblems = Effect.fn("FleetBoard.warnSnapshotProblems") } else { previous.set(root, next); } + checked.add(snapshot); }); diff --git a/packages/work-fleet/src/host/publishedRecords.ts b/packages/work-fleet/src/host/publishedRecords.ts new file mode 100644 index 00000000..f6229f9e --- /dev/null +++ b/packages/work-fleet/src/host/publishedRecords.ts @@ -0,0 +1,41 @@ +import { Effect } from "effect"; +import type { FleetHealth } from "@shivaedev/work-board/server/integration/diagnostics.ts"; +import { views } from "#engine/views.ts"; +import { executing, type FleetRecord } from "#model.ts"; + +type RecordCounts = Pick; + +// The store hands back the same record object until its row changes, so equal identities mean the published views still hold. +function sameRecords(previous: readonly FleetRecord[] | undefined, next: readonly FleetRecord[]) { + return previous !== undefined && previous.length === next.length && previous.every((record, index) => record === next[index]); +} + +export function publishedRecords() { + let published: readonly FleetRecord[] | undefined; + return { + changed: (records: readonly FleetRecord[]) => !sameRecords(published, records), + published: (records: readonly FleetRecord[]) => + Effect.sync(() => { + published = records; + }), + }; +} + +export function recordCounter() { + let counted: { readonly counts: RecordCounts; readonly records: readonly FleetRecord[] } | undefined; + return (records: readonly FleetRecord[]) => { + if (counted === undefined || !sameRecords(counted.records, records)) { + counted = { counts: recordCounts(records), records }; + } + return counted.counts; + }; +} + +function recordCounts(records: readonly FleetRecord[]): RecordCounts { + const { waiting } = views(records); + return { + queued: records.filter((record) => record.step === "queued").length, + running: records.filter(executing).length, + waiting: { automation: waiting.automation.length, maintainer: waiting.maintainer.length, orchestrator: waiting.orchestrator.length }, + }; +} diff --git a/packages/work-fleet/src/host/statusLayer.test.ts b/packages/work-fleet/src/host/statusLayer.test.ts index 41047227..50de2ee5 100644 --- a/packages/work-fleet/src/host/statusLayer.test.ts +++ b/packages/work-fleet/src/host/statusLayer.test.ts @@ -48,3 +48,18 @@ publicationStory.it("a failed publication keeps execution usable and the next sc expect(fleet.publications.status()).toContain("Stage: done"); expect(fleet.publications.task()).toBe(fleet.publications.source()); }); +publicationStory.it("idle passes over unchanged work leave the status unrendered until the work changes", [publishedBoard], function* (fleet) { + yield* fleet.publications.observeWrites(); + yield* fleet.publications.idlePass(); + yield* fleet.publications.idlePass(); + expect(fleet.publications.publishedWorkIds()).toEqual([]); + yield* fleet.prepareWork("one"); + expect(fleet.publications.publishedWorkIds()).toEqual([["one"]]); +}); +publicationStory.it("an idle pass repairs a status write that failed while the work stayed unchanged", [publishedBoard], function* (fleet) { + yield* fleet.publications.blockStatus(); + yield* fleet.prepareWork("one"); + fleet.publications.restoreStatus(); + yield* fleet.publications.idlePass(); + expect(fleet.publications.status()).toContain("/_board/item/one"); +}); diff --git a/packages/work-fleet/src/host/statusLayer.ts b/packages/work-fleet/src/host/statusLayer.ts index 6a1b8cad..4f3358a4 100644 --- a/packages/work-fleet/src/host/statusLayer.ts +++ b/packages/work-fleet/src/host/statusLayer.ts @@ -6,11 +6,10 @@ import { publishAfterBoardPass } from "#board/readPass.ts"; import { writeHealth } from "#board/writeHealth.ts"; import { writeStatus } from "#board/writeStatus.ts"; import { Health, HealthPublication, HealthSnapshot } from "#engine/health.ts"; -import { views } from "#engine/views.ts"; import { Fleet } from "#fleet.ts"; import { GithubApi } from "#host/githubApi.ts"; import type { GithubObservation } from "#host/githubRateLimit.ts"; -import { executing, type FleetRecord } from "#model.ts"; +import { publishedRecords, recordCounter } from "#host/publishedRecords.ts"; const { SourceFailure } = diagnosticsApi; @@ -33,17 +32,22 @@ export function statusLayer( const realRoot = yield* fs.realPath(root); const lock = yield* Semaphore.make(1); const snapshot = yield* Ref.make(null); + const status = publishedRecords(); + const counts = recordCounter(); const publication = Effect.fn("FleetStatus.publication")( function* (read: typeof fleet.list) { const records = (yield* read()).map(({ value }) => value); - yield* writeStatus(root, realRoot, records).pipe( - Effect.catch((error) => Console.error(`work-fleet: status publication failed: ${error.message}`)), - ); + if (status.changed(records)) { + yield* writeStatus(root, realRoot, records).pipe( + Effect.andThen(status.published(records)), + Effect.catch((error) => Console.error(`work-fleet: status publication failed: ${error.message}`)), + ); + } if (healthConfig === undefined) { return; } const state = health._tag === "Some" ? yield* Ref.get(health.value) : { lastCapacityRejection: null, lastSchedulerPass: null }; - const nextHealth = fleetHealth(records, yield* github, { ...(yield* healthConfig), ...state }, yield* Clock.currentTimeMillis); + const nextHealth = fleetHealth(counts(records), yield* github, { ...(yield* healthConfig), ...state }, yield* Clock.currentTimeMillis); yield* Ref.set(snapshot, nextHealth); yield* writeHealth(root, realRoot, nextHealth); }, @@ -108,7 +112,7 @@ export function statusLayer( } function fleetHealth( - records: readonly FleetRecord[], + counts: Pick, observations: readonly GithubObservation[], observed: Pick, observedAt: number, @@ -118,17 +122,10 @@ function fleetHealth( observedAt, version: 2, ...observed, - queued: records.filter((record) => record.step === "queued").length, - running: records.filter(executing).length, - waiting: waitingCounts(records), + ...counts, }; } -function waitingCounts(records: Parameters[0]) { - const { waiting } = views(records); - return { automation: waiting.automation.length, maintainer: waiting.maintainer.length, orchestrator: waiting.orchestrator.length }; -} - function readSnapshot(snapshot: Ref.Ref) { return Ref.get(snapshot).pipe( Effect.flatMap((value) => diff --git a/packages/work-fleet/src/test-support/boardReadStory.ts b/packages/work-fleet/src/test-support/boardReadStory.ts index 1e900a7b..36b0c582 100644 --- a/packages/work-fleet/src/test-support/boardReadStory.ts +++ b/packages/work-fleet/src/test-support/boardReadStory.ts @@ -2,6 +2,7 @@ import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { Context, Deferred, Effect, Fiber, FileSystem, Layer, type Path } from "effect"; import { storyKit } from "@shivaedev/test-story/server/storyKit.ts"; +import type { Snapshot as BoardSnapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; import { Workspace } from "@shivaedev/work-board/server/workspace.ts"; import { briefRevision } from "#board/briefRevision.ts"; import { publishAfterBoardPass, withBoardReadPass, workMetadata } from "#board/readPass.ts"; @@ -27,14 +28,19 @@ export const boardReadStory = storyKit({ } const context = yield* Layer.build(state.fixture.board.pipe(Layer.provideMerge(state.fixture.services))); state.board = Context.get(context, BoardGateway); + state.workspace = Context.get(context, Workspace); state.fixture.services = Layer.succeedContext(context); - yield* state.fixture.observeWith(Context.get(context, Workspace)); + yield* state.fixture.observeWith(state.workspace); }, }, clock: "live", *create() { const fixture = yield* boardReads(0); - const optional: { board: typeof BoardGateway.Service | undefined } = { board: undefined }; + const optional: { + board: typeof BoardGateway.Service | undefined; + indexRead: BoardSnapshot | undefined; + workspace: typeof Workspace.Service | undefined; + } = { board: undefined, indexRead: undefined, workspace: undefined }; const publications: string[] = []; return { ...optional, fixture, fs: yield* FileSystem.FileSystem, publications }; }, @@ -60,6 +66,12 @@ export const boardReadStory = storyKit({ const approvedIds = approvedWorkIds(resolveHost(fixture.root, { database: syntheticDatabase, provider: "local" })).pipe( Effect.provide(fixture.services), ); + function index() { + if (state.workspace === undefined) { + throw new Error("The headless Board workspace did not open"); + } + return state.workspace.index().pipe(Effect.orDie); + } function read(id: string, options?: { readonly fresh?: boolean }) { return Effect.suspend(() => { story.tell(`read Board work ${id}${options?.fresh ? " freshly" : ""}`); @@ -86,6 +98,13 @@ export const boardReadStory = storyKit({ story.tell("defer the generated status publication"); return publishAfterBoardPass(publish); }, + directoriesReadDuring: (effect: Effect.Effect) => + Effect.gen(function* () { + const before = fixture.directoryReads(); + yield* effect; + story.tell("count the directories read meanwhile"); + return fixture.directoryReads() - before; + }), duringOnePass: (effect: Effect.Effect) => { story.tell("start a Board read pass"); return withBoardReadPass(effect).pipe(Effect.provide(fixture.services)); @@ -105,6 +124,11 @@ export const boardReadStory = storyKit({ }, }; }), + laterIndexRead: () => + Effect.map(index(), (snapshot) => { + story.tell("read the Board index again and compare it with the earlier read"); + return snapshot === state.indexRead ? "the same snapshot" : "a rebuilt snapshot"; + }), listingFailure: () => refusal(boardSnapshot(fixture.root, fixture.root).pipe(Effect.provide(fixture.services))), makeReadable: () => { story.tell("every Board file becomes readable again"); @@ -122,6 +146,11 @@ export const boardReadStory = storyKit({ return (yield* fs.exists(path)) ? yield* fs.readFileString(path) : undefined; }), read, + readIndex: () => + Effect.map(index(), (snapshot) => { + story.tell("read the Board index"); + state.indexRead = snapshot; + }), readInPass: (id: string, options?: { readonly fresh?: boolean }) => { story.tell(`read Board work ${id} in a new pass`); return withBoardReadPass(board.get(id, options)).pipe(Effect.provide(fixture.services)); @@ -172,3 +201,9 @@ export function hasBoardItems(count: number) { } }); } + +export function namesMissingArea() { + return boardReadStory.trait("files", "an open Board item names an area folder that does not exist", function* ({ fixture }) { + yield* fixture.write("area.md", "---\nid: item.area\nkind: task\nareas: [missing]\n---\n# Area\n"); + }); +} diff --git a/packages/work-fleet/src/test-support/boardReads.ts b/packages/work-fleet/src/test-support/boardReads.ts index a94221c7..8c49e336 100644 --- a/packages/work-fleet/src/test-support/boardReads.ts +++ b/packages/work-fleet/src/test-support/boardReads.ts @@ -12,6 +12,7 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu const unreadable = new Set(); let workspace: typeof Workspace.Service | undefined; let announce = true; + let directoryReads = 0; function refresh(files: readonly string[]) { return workspace && announce ? workspace.settings().pipe(Effect.flatMap(({ changes }) => changes.refreshPaths?.(files) ?? Effect.void)) @@ -21,9 +22,12 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu const tracked: FileSystem.FileSystem = { ...fs, readDirectory: (folder) => - unreadable.has(path.relative(root, folder) || ".") - ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readDirectory", module: "FileSystem" })) - : fs.readDirectory(folder), + Effect.suspend(() => { + directoryReads += 1; + return unreadable.has(path.relative(root, folder) || ".") + ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readDirectory", module: "FileSystem" })) + : fs.readDirectory(folder); + }), readFile: (file) => unreadable.has(path.relative(root, file)) ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readFile", module: "FileSystem" })) @@ -62,6 +66,7 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu Layer.provide(services), Layer.provide(Layer.succeed(Clock.Clock, Clock.Clock.defaultValue())), ), + directoryReads: () => directoryReads, observeWith: (board: typeof Workspace.Service) => Effect.sync(() => { workspace = board;