diff --git a/.changeset/quiet-board-rebuilds.md b/.changeset/quiet-board-rebuilds.md new file mode 100644 index 00000000..f91ccbf0 --- /dev/null +++ b/.changeset/quiet-board-rebuilds.md @@ -0,0 +1,10 @@ +--- +"@shivaedev/work-board": patch +"@shivaedev/work-fleet": patch +--- + +### Fixed + +- `@shivaedev/work-board`: The Board index keeps the decoded settled catalogue and archive state until those files change, so a reported edit reads only the edited file instead of reading the whole catalogue again on every rebuild. +- `@shivaedev/work-fleet`: A fresh read of a brief whose source did not change no longer invalidates the Board index, so reconciling done work no longer rebuilds the index for each record. +- `@shivaedev/work-fleet`: Fleet renders `fleet/status.md` and counts health only when its records change, and checks each Board snapshot's problems once instead of on every read pass. diff --git a/packages/work-board/src/server/archive/documents.ts b/packages/work-board/src/server/archive/documents.ts index c57ddb09..57060303 100644 --- a/packages/work-board/src/server/archive/documents.ts +++ b/packages/work-board/src/server/archive/documents.ts @@ -1,5 +1,6 @@ -import { Effect, FileSystem, Order, Path, Schema } from "effect"; +import { Effect, FileSystem, Order, Path, type PlatformError, Schema } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; +import { cachedRead } from "#server/files/cachedRead.ts"; import { type Changes, refreshWritten } from "#server/files/changes.ts"; const RETENTION = 14 * 24 * 60 * 60 * 1000; @@ -23,16 +24,34 @@ function identity(document: MetadataDocument) { return document.parsed.fields.id ?? document.file.replace(/^(?:tasks|archive)\//u, ""); } +export interface ArchiveState { + readonly restored: ReadonlySet; + readonly saved: string; +} + +const STATE_FILE = ".board-archive.json"; + +function decodeState(saved: string) { + return Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe( + Effect.orDie, + Effect.map((identities): ArchiveState => ({ restored: new Set(identities), saved })), + ); +} + +export function archiveStateReader(root: string) { + return cachedRead(root, STATE_FILE, { restored: new Set(), saved: "[]" }, decodeState); +} + export const archiveDocuments = Effect.fn("WorkBoard.archiveDocuments")(function* ( root: string, documents: readonly MetadataDocument[], now: number, + state?: Effect.Effect, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const stateFile = path.join(root, ".board-archive.json"); - const saved = (yield* fs.exists(stateFile)) ? yield* fs.readFileString(stateFile) : "[]"; - const restored = new Set(yield* Schema.decodeUnknownEffect(ArchivedIdentities)(saved).pipe(Effect.orDie)); + const stateFile = path.join(root, STATE_FILE); + const { restored, saved } = yield* state ?? archiveStateReader(root); const archived = new Set(documents.filter((document) => document.file.startsWith("archive/")).map(identity)); const moved: string[] = []; for (const document of documents) { diff --git a/packages/work-board/src/server/archive/loadSettled.ts b/packages/work-board/src/server/archive/loadSettled.ts index 93cc6dd4..0d3531bf 100644 --- a/packages/work-board/src/server/archive/loadSettled.ts +++ b/packages/work-board/src/server/archive/loadSettled.ts @@ -1,12 +1,17 @@ -import { Effect, FileSystem, Path } from "effect"; +import { Effect, FileSystem, Path, type PlatformError } from "effect"; import { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import { documentSource, metadataModel } from "#server/metadata/model.ts"; import { readDocument } from "#server/search/readDocument.ts"; import type { Snapshot } from "#server/search/snapshot.ts"; -import { readCatalogue, recordPath } from "./settledCatalogue.ts"; +import { catalogueReader, readCatalogue, recordPath, type SettledCatalogue } from "./settledCatalogue.ts"; -export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* (root: string, snapshot: Snapshot, selection: string) { - const catalogue = yield* readCatalogue(root); +export const loadSettled = Effect.fn("WorkBoard.loadSettled")(function* ( + root: string, + snapshot: Snapshot, + selection: string, + settled?: Effect.Effect, +) { + const { records: catalogue } = yield* settled ?? catalogueReader(root); const groups = new Set(catalogue.filter((record) => record.id === selection || record.file === selection).map((record) => record.item)); groups.add(selection); const selected = snapshot.documents.find((document) => document.file === selection)?.parsed.fields.id; diff --git a/packages/work-board/src/server/archive/settled.ts b/packages/work-board/src/server/archive/settled.ts index 4e6e46f9..1eb60ce0 100644 --- a/packages/work-board/src/server/archive/settled.ts +++ b/packages/work-board/src/server/archive/settled.ts @@ -1,7 +1,7 @@ -import { Effect, FileSystem, Path } from "effect"; +import { Effect, FileSystem, Path, type PlatformError } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; import { documentSource } from "#server/metadata/model.ts"; -import { appendCatalogue, readCatalogue, recordPath, type SettledRecord } from "./settledCatalogue.ts"; +import { appendCatalogue, catalogueReader, recordPath, type SettledCatalogue, type SettledRecord } from "./settledCatalogue.ts"; import { settledPlan } from "./settledPlan.ts"; import { settledRecords } from "./settledRecords.ts"; @@ -37,7 +37,11 @@ const moveSettled = Effect.fn("WorkBoard.moveSettled")( ), ); -export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (root: string, documents: readonly MetadataDocument[]) { +export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* ( + root: string, + documents: readonly MetadataDocument[], + settled?: Effect.Effect, +) { const ids = new Set(documents.map((document) => document.parsed.fields.id)); const pending = documents.some( ({ parsed: { fields } }) => @@ -48,10 +52,10 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r if (!pending && settledRecords(documents).length === 0) { return []; } - const catalogue = yield* readCatalogue(root); + const catalogue = yield* settled ?? catalogueReader(root); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const planned = yield* Effect.filter(settledPlan(documents, catalogue), ({ document }) => + const planned = yield* Effect.filter(settledPlan(documents, catalogue.byId), ({ document }) => fs.readFileString(path.join(root, document.file)).pipe( Effect.map((source) => source === documentSource(document)), Effect.catchIf( @@ -63,8 +67,7 @@ export const archiveSettled = Effect.fn("WorkBoard.archiveSettled")(function* (r if (planned.length === 0) { return []; } - const existing = new Set(catalogue.map((record) => record.id)); - const added = planned.map(({ record }) => record).filter((record) => !existing.has(record.id)); + const added = planned.map(({ record }) => record).filter((record) => !catalogue.byId.has(record.id)); // Publish locations before removing sources, so an interrupted move remains discoverable and can finish on the next read. yield* appendCatalogue(root, added); return (yield* Effect.forEach(planned, ({ document, record }) => moveSettled(root, document, record))).flat(); diff --git a/packages/work-board/src/server/archive/settledCatalogue.ts b/packages/work-board/src/server/archive/settledCatalogue.ts index f190260d..21014ee0 100644 --- a/packages/work-board/src/server/archive/settledCatalogue.ts +++ b/packages/work-board/src/server/archive/settledCatalogue.ts @@ -1,4 +1,5 @@ import { Effect, FileSystem, Path, Schema } from "effect"; +import { cachedRead } from "#server/files/cachedRead.ts"; const Record = Schema.Struct({ file: Schema.String, @@ -11,22 +12,40 @@ const Record = Schema.Struct({ const CatalogueEntry = Schema.fromJsonString(Record); export type SettledRecord = typeof Record.Type; +export interface SettledCatalogue { + readonly byId: ReadonlyMap; + readonly files: ReadonlyMap; + readonly records: readonly SettledRecord[]; +} + +const CATALOGUE = ".board-settled/catalogue.jsonl"; +const EMPTY: SettledCatalogue = { byId: new Map(), files: new Map(), records: [] }; + +const decodeCatalogue = Effect.fn("WorkBoard.decodeCatalogue")(function* (source: string) { + const lines = source.split("\n").filter((line) => line.length > 0); + const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie)); + const byId = new Map(records.map((record) => [record.id, record])); + return { byId, files: new Map([...byId].map(([id, record]) => [id, record.file])), records: [...byId.values()] } satisfies SettledCatalogue; +}); + export const readCatalogue = Effect.fn("WorkBoard.readCatalogue")(function* (root: string) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const file = path.join(root, ".board-settled/catalogue.jsonl"); + const file = path.join(root, CATALOGUE); if (!(yield* fs.exists(file))) { return []; } - const lines = (yield* fs.readFileString(file)).split("\n").filter((line) => line.length > 0); - const records = yield* Effect.forEach(lines, (line) => Schema.decodeUnknownEffect(CatalogueEntry)(line).pipe(Effect.orDie)); - return [...new Map(records.map((record) => [record.id, record])).values()]; + return (yield* decodeCatalogue(yield* fs.readFileString(file))).records; }); +export function catalogueReader(root: string) { + return cachedRead(root, CATALOGUE, EMPTY, decodeCatalogue); +} + export const appendCatalogue = Effect.fn("WorkBoard.appendCatalogue")(function* (root: string, records: readonly SettledRecord[]) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const file = path.join(root, ".board-settled/catalogue.jsonl"); + const file = path.join(root, CATALOGUE); yield* fs.makeDirectory(path.dirname(file), { recursive: true }); for (const record of records) { yield* fs.writeFileString(file, `${JSON.stringify(record)}\n`, { flag: "a" }); diff --git a/packages/work-board/src/server/archive/settledPlan.ts b/packages/work-board/src/server/archive/settledPlan.ts index d66b1618..a8d36902 100644 --- a/packages/work-board/src/server/archive/settledPlan.ts +++ b/packages/work-board/src/server/archive/settledPlan.ts @@ -4,8 +4,7 @@ import { revisionOf } from "#server/responses/records.ts"; import type { SettledRecord } from "./settledCatalogue.ts"; import { settledRecords } from "./settledRecords.ts"; -export function settledPlan(documents: readonly MetadataDocument[], catalogue: readonly SettledRecord[]) { - const existing = new Map(catalogue.map((record) => [record.id, record])); +export function settledPlan(documents: readonly MetadataDocument[], existing: ReadonlyMap) { const candidates = new Map(settledRecords(documents).map((candidate) => [candidate.id, candidate])); return documents.flatMap((document) => { const id = document.parsed.fields.id ?? ""; diff --git a/packages/work-board/src/server/files/cachedRead.ts b/packages/work-board/src/server/files/cachedRead.ts new file mode 100644 index 00000000..efa866b9 --- /dev/null +++ b/packages/work-board/src/server/files/cachedRead.ts @@ -0,0 +1,32 @@ +import { Effect, FileSystem, Option, Path } from "effect"; + +function version(info: FileSystem.File.Info) { + return [info.dev, Option.getOrUndefined(info.ino), info.size, Option.getOrUndefined(Option.map(info.mtime, (date) => date.getTime()))].join(":"); +} + +// Every rebuild consults these bookkeeping files, so their decoded value is kept until the file's identity, size or modification time moves. +export function cachedRead(root: string, name: string, missing: A, decode: (source: string) => Effect.Effect) { + let cached: { readonly value: A; readonly version: string } | undefined; + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const file = (yield* Path.Path).join(root, name); + const info = yield* fs.stat(file).pipe( + Effect.map(Option.some), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + ); + if (Option.isNone(info)) { + cached = undefined; + return missing; + } + const current = version(info.value); + if (cached?.version === current) { + return cached.value; + } + const value = yield* decode(yield* fs.readFileString(file)); + cached = { value, version: current }; + return value; + }); +} diff --git a/packages/work-board/src/server/metadata/updateModel.ts b/packages/work-board/src/server/metadata/updateModel.ts index f42d8449..5489e4fe 100644 --- a/packages/work-board/src/server/metadata/updateModel.ts +++ b/packages/work-board/src/server/metadata/updateModel.ts @@ -3,6 +3,9 @@ import { metadataProblems } from "./problems.ts"; import { referencesOf } from "./referencesOf.ts"; function changedSettledIds(before: ReadonlyMap | undefined, after: ReadonlyMap | undefined) { + if (before === after) { + return []; + } return [...new Set([...(before?.keys() ?? []), ...(after?.keys() ?? [])])].filter((id) => before?.get(id) !== after?.get(id)); } diff --git a/packages/work-board/src/server/search/archiveInventory.ts b/packages/work-board/src/server/search/archiveInventory.ts index 27cc213e..368e5a81 100644 --- a/packages/work-board/src/server/search/archiveInventory.ts +++ b/packages/work-board/src/server/search/archiveInventory.ts @@ -1,7 +1,13 @@ -import { Effect } from "effect"; +import { Effect, type FileSystem, type Path, type PlatformError } from "effect"; import type { MetadataDocument } from "#integration/metadata.ts"; -import { archiveDocuments } from "#server/archive/documents.ts"; +import { type ArchiveState, archiveDocuments } from "#server/archive/documents.ts"; import { archiveSettled } from "#server/archive/settled.ts"; +import type { SettledCatalogue } from "#server/archive/settledCatalogue.ts"; + +export interface ArchiveReads { + readonly catalogue: Effect.Effect; + readonly state: Effect.Effect; +} export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function* ( root: string, @@ -9,6 +15,7 @@ export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function modified: ReadonlyMap, now: number, unavailable: readonly string[], + reads: ArchiveReads, ) { if (unavailable.length > 0) { return []; @@ -17,6 +24,7 @@ export const archiveInventory = Effect.fn("WorkBoard.archiveInventory")(function root, documents.map((document) => ({ ...document, modified: modified.get(document.file) ?? now })), now, + reads.state, ); - return [...moved, ...(yield* archiveSettled(root, documents))]; + return [...moved, ...(yield* archiveSettled(root, documents, reads.catalogue))]; }); diff --git a/packages/work-board/src/server/search/readCosts.spec.ts b/packages/work-board/src/server/search/readCosts.spec.ts index c4cbb95d..97c501a7 100644 --- a/packages/work-board/src/server/search/readCosts.spec.ts +++ b/packages/work-board/src/server/search/readCosts.spec.ts @@ -1,6 +1,7 @@ import { expect } from "vitest"; import { outcomes } from "@shivaedev/test-story/server/outcomes.ts"; import { indexedAliases } from "#test/indexedAliases.ts"; +import { indexedSettled } from "#test/indexedSettled.ts"; import { indexedFiles, indexStory } from "#test/indexStory.ts"; const NOTES = 100; @@ -30,3 +31,29 @@ indexStory.it( ]); }, ); + +indexStory.it( + "a reported edit reads only the edited file while the settled catalogue and archive state stand unchanged", + [indexedFiles({ "a.md": "---\nid: a\n---\n# Alpha", "b.md": "# Beta" }), indexedSettled(["settled-one", "settled-two"])], + function* ({ editorWrites, filesRead, settledIds, watcherReports }) { + yield* editorWrites("a.md", "---\nid: a\n---\n# Alpha, revised"); + yield* watcherReports("a.md"); + + yield* outcomes([ + ["files read", () => filesRead(), ["a.md"]], + ["settled records", () => settledIds(), ["settled-one", "settled-two"]], + ]); + }, +); + +indexStory.it( + "the next reported change picks up a record another process appended to the settled catalogue", + [indexedFiles({ "a.md": "# Alpha" }), indexedSettled(["settled-one"])], + function* ({ anotherProcessSettles, editorWrites, settledIds, watcherReports }) { + yield* anotherProcessSettles("settled-two"); + yield* editorWrites("a.md", "# Alpha, revised"); + yield* watcherReports("a.md"); + + expect(yield* settledIds()).toEqual(["settled-one", "settled-two"]); + }, +); diff --git a/packages/work-board/src/server/search/snapshot.ts b/packages/work-board/src/server/search/snapshot.ts index cae18527..2e4390b4 100644 --- a/packages/work-board/src/server/search/snapshot.ts +++ b/packages/work-board/src/server/search/snapshot.ts @@ -2,16 +2,16 @@ import { Clock, Context, Effect, FileSystem, Option, Path, type PlatformError, R import type { InternalServerError } from "@shivaedev/errors/taxonomy.ts"; import type { MetadataDocument } from "#integration/metadata.ts"; import type { Indexing } from "#rpc/Indexing.ts"; -import { refreshArchived } from "#server/archive/documents.ts"; +import { archiveStateReader, refreshArchived } from "#server/archive/documents.ts"; import { loadSettled } from "#server/archive/loadSettled.ts"; -import { readCatalogue } from "#server/archive/settledCatalogue.ts"; +import { catalogueReader } from "#server/archive/settledCatalogue.ts"; import type { Changes } from "#server/files/changes.ts"; import { invalidationsSince } from "#server/files/invalidations.ts"; import type { MarkdownFile } from "#server/files/list.ts"; import { markdownPaths } from "#server/files/markdownPath.ts"; import type { metadataModel } from "#server/metadata/model.ts"; import { updateModel } from "#server/metadata/updateModel.ts"; -import { archiveInventory } from "./archiveInventory.ts"; +import { type ArchiveReads, archiveInventory } from "./archiveInventory.ts"; import type { Entry } from "./entries.ts"; import { fileCounter } from "./fileCounter.ts"; import { type IndexedDocument, readDocument } from "./readDocument.ts"; @@ -73,6 +73,7 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( let records = new Map(); let listed = new Map(); const enrich = updateEntries(); + const archiveReads: ArchiveReads = { catalogue: catalogueReader(root), state: archiveStateReader(root) }; function find(file: MarkdownFile, dirty: ReadonlySet | undefined, resolve: (file: string) => Effect.Effect) { const previous = records.get(file.path); return shouldRead(file, previous, listed, dirty) @@ -113,19 +114,19 @@ export const searchSnapshot = Effect.fn("WorkBoard.searchSnapshot")(function* ( } } const modified = new Map(files.map((file) => [file.path, file.modified])); - const moved = yield* archiveInventory(root, documents, modified, now, unavailable); + const moved = yield* archiveInventory(root, documents, modified, now, unavailable, archiveReads); housekeptAt = now; if (moved.length > 0) { yield* refreshArchived(changes, moved); return yield* buildSnapshot(hit); } - const settled = new Map((yield* readCatalogue(root)).map((record) => [record.id, record.file])); + const { files: settled } = yield* archiveReads.catalogue; const model = updateModel(hit?.model, documents, unavailable, home ?? files[0]?.path, settled); const enriched = enrich(next, model); // Only the file services: the fiber context holds this call's stack frame, which retains the previous snapshot. const services = (yield* Effect.context()).pipe(Context.pick(FileSystem.FileSystem, Path.Path)); const snapshot: Snapshot = { - archived: (selection) => loadSettled(root, snapshot, selection).pipe(Effect.provideContext(services)), + archived: (selection) => loadSettled(root, snapshot, selection, archiveReads.catalogue).pipe(Effect.provideContext(services)), documents, entries: enriched, maintainer, diff --git a/packages/work-board/src/server/workspace.ts b/packages/work-board/src/server/workspace.ts index 32f18c27..1029175b 100644 --- a/packages/work-board/src/server/workspace.ts +++ b/packages/work-board/src/server/workspace.ts @@ -93,7 +93,8 @@ export const Workspace = defineService({ } return { ...metadata, - archived: (selection: string) => Effect.flatMap(index, (snapshot) => loadSettled(options.root, snapshot, selection)), + archived: (selection: string) => + Effect.flatMap(index, (snapshot) => snapshot.archived?.(selection) ?? loadSettled(options.root, snapshot, selection)), document, index: () => index, settings: () => Effect.succeed(settings), diff --git a/packages/work-board/src/test-support/indexStory.ts b/packages/work-board/src/test-support/indexStory.ts index c9a8d62c..918c5cff 100644 --- a/packages/work-board/src/test-support/indexStory.ts +++ b/packages/work-board/src/test-support/indexStory.ts @@ -1,4 +1,4 @@ -import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs"; +import { appendFileSync, existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs"; import { join, relative } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { NodeServices } from "@effect/platform-node"; @@ -12,6 +12,7 @@ import { collectGarbage } from "./collectGarbage.ts"; import { incrementalIndex } from "./incrementalIndex.ts"; import { comparable, kept } from "./indexComparison.ts"; import { indexState } from "./indexState.ts"; +import { SETTLED_CATALOGUE, settledLine } from "./settledCatalogue.ts"; interface IndexEngine { addDuringListing: boolean; @@ -87,6 +88,11 @@ export const indexStory = storyKit({ return engine.before; } return { + anotherProcessSettles: (id: string) => + Effect.sync(() => { + story.tell(`another process records ${id} in the settled catalogue`); + appendFileSync(join(engine.notes.root, SETTLED_CATALOGUE), settledLine(id)); + }), bodyOf: (file: string) => Effect.map(current(), (snapshot) => { story.tell(`the reader reads the indexed body of ${file}`); @@ -241,6 +247,11 @@ export const indexStory = storyKit({ story.tell(`the reader searches the index for ${JSON.stringify(query)} and reads the title of ${id}`); return searchMatch(snapshot.entries, query).results.find((entry) => entry.id === id)?.title ?? "not found"; }), + settledIds: () => + Effect.map(current(), (snapshot) => { + story.tell("the reader reads which settled records the index knows"); + return [...(snapshot.model.settled?.keys() ?? [])].sort(Order.String); + }), unavailableFiles: () => Effect.map(current(), (snapshot) => { story.tell("the reader reads which files the index could not read"); diff --git a/packages/work-board/src/test-support/indexedSettled.ts b/packages/work-board/src/test-support/indexedSettled.ts new file mode 100644 index 00000000..d011c99c --- /dev/null +++ b/packages/work-board/src/test-support/indexedSettled.ts @@ -0,0 +1,9 @@ +import { indexStory } from "./indexStory.ts"; +import { SETTLED_CATALOGUE, settledLine } from "./settledCatalogue.ts"; + +export function indexedSettled(ids: readonly string[]) { + return indexStory.trait("project", `the board keeps settled records for ${ids.join(", ")} and an archive state`, (engine) => { + engine.notes.write(SETTLED_CATALOGUE, ids.map(settledLine).join("")); + engine.notes.write(".board-archive.json", "[]"); + }); +} diff --git a/packages/work-board/src/test-support/servingBoardVerbs.ts b/packages/work-board/src/test-support/servingBoardVerbs.ts index 13a63dd3..e6d2e642 100644 --- a/packages/work-board/src/test-support/servingBoardVerbs.ts +++ b/packages/work-board/src/test-support/servingBoardVerbs.ts @@ -14,6 +14,8 @@ import type { ServingOptions } from "./servingEngine.ts"; import { servingObservations } from "./servingObservations.ts"; import { type BoardEngine, connected, served } from "./storyEngine.ts"; +const BOARD_OWNED = new Set(["board.sock", ".board-archive.json", ".board-settled/catalogue.jsonl"]); + export function servingBoardVerbs(engine: BoardEngine, tell: (line: string) => void) { const { documentAt, linked, changeSeen, launched, pluginBoard, pluginTools } = servingBoardReads(engine, tell); return { @@ -247,13 +249,13 @@ export function servingBoardVerbs(engine: BoardEngine, tell: (line: string) => v }, visitedPaths: () => Effect.sync(() => { - tell("the reader lists every workspace path the board visited, apart from its own socket"); + tell("the reader lists every workspace path the board visited, apart from its own socket and archive bookkeeping files"); const paths = servingObservations(engine).paths; if (!paths) { throw new Error("Add the directoryReadsRecorded trait before listing visited paths"); } return [...new Set(paths.visited.map((path) => relative(engine.notes.root, path)))] - .filter((path) => path !== "board.sock") + .filter((path) => !BOARD_OWNED.has(path)) .sort(Order.String); }), }; diff --git a/packages/work-board/src/test-support/settledCatalogue.ts b/packages/work-board/src/test-support/settledCatalogue.ts new file mode 100644 index 00000000..b86f8908 --- /dev/null +++ b/packages/work-board/src/test-support/settledCatalogue.ts @@ -0,0 +1,5 @@ +export const SETTLED_CATALOGUE = ".board-settled/catalogue.jsonl"; + +export function settledLine(id: string) { + return `${JSON.stringify({ file: `questions/${id}.md`, id, item: id, modified: 0, revision: "settled", targets: [] })}\n`; +} diff --git a/packages/work-fleet/quality-baseline.jsonl b/packages/work-fleet/quality-baseline.jsonl index d1dd7348..80c1761c 100644 --- a/packages/work-fleet/quality-baseline.jsonl +++ b/packages/work-fleet/quality-baseline.jsonl @@ -152,7 +152,7 @@ {"path":"packages/work-fleet/src/host/reportQuestion.spec.ts","rule":"effect/boundary","count":1} {"path":"packages/work-fleet/src/host/runtime.ts","rule":"biome/plugin","count":2} {"path":"packages/work-fleet/src/host/settledAnswers.spec.ts","rule":"effect/boundary","count":1} -{"path":"packages/work-fleet/src/host/statusLayer.ts","rule":"effect/service-definition-assembly","count":6} +{"path":"packages/work-fleet/src/host/statusLayer.ts","rule":"effect/service-definition-assembly","count":5} {"path":"packages/work-fleet/src/host/supervision.spec.ts","rule":"effect/boundary","count":7} {"path":"packages/work-fleet/src/host/supervisor.ts","rule":"effect/boundary","count":1} {"path":"packages/work-fleet/src/host/supervisorPort.ts","rule":"effect/boundary","count":1} diff --git a/packages/work-fleet/src/board/freshRead.spec.ts b/packages/work-fleet/src/board/freshRead.spec.ts index 2fbb5558..21bfa72a 100644 --- a/packages/work-fleet/src/board/freshRead.spec.ts +++ b/packages/work-fleet/src/board/freshRead.spec.ts @@ -1,7 +1,7 @@ import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { outcomes } from "@shivaedev/test-story/server/outcomes.ts"; -import { boardReadStory, hasBoardItems } from "#test/boardReadStory.ts"; +import { boardReadStory, hasBoardItems, namesMissingArea } from "#test/boardReadStory.ts"; import { watchedBoardReads } from "#test/watchedBoardReads.ts"; boardReadStory.it( @@ -70,3 +70,26 @@ boardReadStory.it( ); }, ); + +boardReadStory.it( + "a fresh read of an unchanged brief keeps the Board index without a rebuild", + [hasBoardItems(2)], + function* ({ laterIndexRead, readIndex, readInPass }) { + yield* readInPass("item.0"); + yield* readIndex(); + + yield* readInPass("item.0", { fresh: true }); + + expect(yield* laterIndexRead()).toBe("the same snapshot"); + }, +); + +boardReadStory.it( + "a later read pass over the same Board index checks no area folders again", + [hasBoardItems(1), namesMissingArea()], + function* ({ directoriesReadDuring, readInPass }) { + yield* readInPass("item.0"); + + expect(yield* directoriesReadDuring(readInPass("item.0"))).toBe(0); + }, +); diff --git a/packages/work-fleet/src/board/readWork.ts b/packages/work-fleet/src/board/readWork.ts index 4bc4b889..2f6190ba 100644 --- a/packages/work-fleet/src/board/readWork.ts +++ b/packages/work-fleet/src/board/readWork.ts @@ -1,4 +1,5 @@ import { Effect, Path, Schema } from "effect"; +import type { MetadataDocument } from "@shivaedev/work-board/integration/metadata.ts"; import { documentSource } from "@shivaedev/work-board/server/metadata/model.ts"; import type { Snapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; import { Workspace } from "@shivaedev/work-board/server/workspace.ts"; @@ -9,6 +10,29 @@ import { warnSnapshotProblems } from "#board/warnSnapshotProblems.ts"; import { metadataProblem, workDocument } from "#board/workPolicy.ts"; import { BoardFailure, Priority } from "#policy.ts"; +const freshDocument = Effect.fn("FleetBoard.freshDocument")(function* (realRoot: string, indexed: MetadataDocument, workId: string) { + const workspace = yield* Workspace; + const fresh = yield* workspace + .metadata(indexed.file) + .pipe( + Effect.mapError( + (error) => + new BoardFailure({ code: "incomplete-observation", message: `Board source could not be read: ${indexed.file} (${error.message})` }), + ), + ); + const problem = metadataProblem(fresh); + if (problem) { + return yield* Effect.fail(problem); + } + if (fresh.parsed.fields.id !== workId) { + return yield* Effect.fail(new BoardFailure({ code: "board-item-missing", message: `Board item no longer exists: ${workId}`, workId })); + } + if (documentSource(fresh) !== documentSource(indexed)) { + yield* recordBoardWrite(realRoot, fresh.file); + } + return fresh; +}); + export const readWork = Effect.fn("FleetBoard.readWork")(function* (current: Snapshot, workId: string, options?: { readonly fresh?: boolean }) { const workspace = yield* Workspace; const path = yield* Path.Path; @@ -24,23 +48,7 @@ export const readWork = Effect.fn("FleetBoard.readWork")(function* (current: Sna } let found = yield* workDocument(root, observed, workId); if (options?.fresh) { - const fresh = yield* workspace - .metadata(found.file) - .pipe( - Effect.mapError( - (error) => - new BoardFailure({ code: "incomplete-observation", message: `Board source could not be read: ${found.file} (${error.message})` }), - ), - ); - const problem = metadataProblem(fresh); - if (problem) { - return yield* Effect.fail(problem); - } - if (fresh.parsed.fields.id !== workId) { - return yield* Effect.fail(new BoardFailure({ code: "board-item-missing", message: `Board item no longer exists: ${workId}`, workId })); - } - found = fresh; - yield* recordBoardWrite(realRoot, fresh.file); + found = yield* freshDocument(realRoot, found, workId); } const deliverable = fleetDeliverable(found.parsed); if (deliverable !== undefined && deliverable !== "report") { diff --git a/packages/work-fleet/src/board/warnSnapshotProblems.ts b/packages/work-fleet/src/board/warnSnapshotProblems.ts index ee84db5d..3e052cf8 100644 --- a/packages/work-fleet/src/board/warnSnapshotProblems.ts +++ b/packages/work-fleet/src/board/warnSnapshotProblems.ts @@ -7,8 +7,13 @@ import { metadataProblem, snapshotProblems } from "#board/workPolicy.ts"; import { BoardFailure } from "#policy.ts"; const previous = new Map>(); +// A snapshot never changes, and every read pass consults it, so its problems are checked once; an area folder created outside the Board shows on the next snapshot. +const checked = new WeakSet(); export const warnSnapshotProblems = Effect.fn("FleetBoard.warnSnapshotProblems")(function* (root: string, snapshot: BoardSnapshot) { + if (checked.has(snapshot)) { + return; + } const seen = previous.get(root); const next = new Map(); const problems = snapshotProblems(snapshot); @@ -38,4 +43,5 @@ export const warnSnapshotProblems = Effect.fn("FleetBoard.warnSnapshotProblems") } else { previous.set(root, next); } + checked.add(snapshot); }); diff --git a/packages/work-fleet/src/host/publishedRecords.ts b/packages/work-fleet/src/host/publishedRecords.ts new file mode 100644 index 00000000..f6229f9e --- /dev/null +++ b/packages/work-fleet/src/host/publishedRecords.ts @@ -0,0 +1,41 @@ +import { Effect } from "effect"; +import type { FleetHealth } from "@shivaedev/work-board/server/integration/diagnostics.ts"; +import { views } from "#engine/views.ts"; +import { executing, type FleetRecord } from "#model.ts"; + +type RecordCounts = Pick; + +// The store hands back the same record object until its row changes, so equal identities mean the published views still hold. +function sameRecords(previous: readonly FleetRecord[] | undefined, next: readonly FleetRecord[]) { + return previous !== undefined && previous.length === next.length && previous.every((record, index) => record === next[index]); +} + +export function publishedRecords() { + let published: readonly FleetRecord[] | undefined; + return { + changed: (records: readonly FleetRecord[]) => !sameRecords(published, records), + published: (records: readonly FleetRecord[]) => + Effect.sync(() => { + published = records; + }), + }; +} + +export function recordCounter() { + let counted: { readonly counts: RecordCounts; readonly records: readonly FleetRecord[] } | undefined; + return (records: readonly FleetRecord[]) => { + if (counted === undefined || !sameRecords(counted.records, records)) { + counted = { counts: recordCounts(records), records }; + } + return counted.counts; + }; +} + +function recordCounts(records: readonly FleetRecord[]): RecordCounts { + const { waiting } = views(records); + return { + queued: records.filter((record) => record.step === "queued").length, + running: records.filter(executing).length, + waiting: { automation: waiting.automation.length, maintainer: waiting.maintainer.length, orchestrator: waiting.orchestrator.length }, + }; +} diff --git a/packages/work-fleet/src/host/statusLayer.test.ts b/packages/work-fleet/src/host/statusLayer.test.ts index 41047227..50de2ee5 100644 --- a/packages/work-fleet/src/host/statusLayer.test.ts +++ b/packages/work-fleet/src/host/statusLayer.test.ts @@ -48,3 +48,18 @@ publicationStory.it("a failed publication keeps execution usable and the next sc expect(fleet.publications.status()).toContain("Stage: done"); expect(fleet.publications.task()).toBe(fleet.publications.source()); }); +publicationStory.it("idle passes over unchanged work leave the status unrendered until the work changes", [publishedBoard], function* (fleet) { + yield* fleet.publications.observeWrites(); + yield* fleet.publications.idlePass(); + yield* fleet.publications.idlePass(); + expect(fleet.publications.publishedWorkIds()).toEqual([]); + yield* fleet.prepareWork("one"); + expect(fleet.publications.publishedWorkIds()).toEqual([["one"]]); +}); +publicationStory.it("an idle pass repairs a status write that failed while the work stayed unchanged", [publishedBoard], function* (fleet) { + yield* fleet.publications.blockStatus(); + yield* fleet.prepareWork("one"); + fleet.publications.restoreStatus(); + yield* fleet.publications.idlePass(); + expect(fleet.publications.status()).toContain("/_board/item/one"); +}); diff --git a/packages/work-fleet/src/host/statusLayer.ts b/packages/work-fleet/src/host/statusLayer.ts index 6a1b8cad..4f3358a4 100644 --- a/packages/work-fleet/src/host/statusLayer.ts +++ b/packages/work-fleet/src/host/statusLayer.ts @@ -6,11 +6,10 @@ import { publishAfterBoardPass } from "#board/readPass.ts"; import { writeHealth } from "#board/writeHealth.ts"; import { writeStatus } from "#board/writeStatus.ts"; import { Health, HealthPublication, HealthSnapshot } from "#engine/health.ts"; -import { views } from "#engine/views.ts"; import { Fleet } from "#fleet.ts"; import { GithubApi } from "#host/githubApi.ts"; import type { GithubObservation } from "#host/githubRateLimit.ts"; -import { executing, type FleetRecord } from "#model.ts"; +import { publishedRecords, recordCounter } from "#host/publishedRecords.ts"; const { SourceFailure } = diagnosticsApi; @@ -33,17 +32,22 @@ export function statusLayer( const realRoot = yield* fs.realPath(root); const lock = yield* Semaphore.make(1); const snapshot = yield* Ref.make(null); + const status = publishedRecords(); + const counts = recordCounter(); const publication = Effect.fn("FleetStatus.publication")( function* (read: typeof fleet.list) { const records = (yield* read()).map(({ value }) => value); - yield* writeStatus(root, realRoot, records).pipe( - Effect.catch((error) => Console.error(`work-fleet: status publication failed: ${error.message}`)), - ); + if (status.changed(records)) { + yield* writeStatus(root, realRoot, records).pipe( + Effect.andThen(status.published(records)), + Effect.catch((error) => Console.error(`work-fleet: status publication failed: ${error.message}`)), + ); + } if (healthConfig === undefined) { return; } const state = health._tag === "Some" ? yield* Ref.get(health.value) : { lastCapacityRejection: null, lastSchedulerPass: null }; - const nextHealth = fleetHealth(records, yield* github, { ...(yield* healthConfig), ...state }, yield* Clock.currentTimeMillis); + const nextHealth = fleetHealth(counts(records), yield* github, { ...(yield* healthConfig), ...state }, yield* Clock.currentTimeMillis); yield* Ref.set(snapshot, nextHealth); yield* writeHealth(root, realRoot, nextHealth); }, @@ -108,7 +112,7 @@ export function statusLayer( } function fleetHealth( - records: readonly FleetRecord[], + counts: Pick, observations: readonly GithubObservation[], observed: Pick, observedAt: number, @@ -118,17 +122,10 @@ function fleetHealth( observedAt, version: 2, ...observed, - queued: records.filter((record) => record.step === "queued").length, - running: records.filter(executing).length, - waiting: waitingCounts(records), + ...counts, }; } -function waitingCounts(records: Parameters[0]) { - const { waiting } = views(records); - return { automation: waiting.automation.length, maintainer: waiting.maintainer.length, orchestrator: waiting.orchestrator.length }; -} - function readSnapshot(snapshot: Ref.Ref) { return Ref.get(snapshot).pipe( Effect.flatMap((value) => diff --git a/packages/work-fleet/src/test-support/boardReadStory.ts b/packages/work-fleet/src/test-support/boardReadStory.ts index 1e900a7b..36b0c582 100644 --- a/packages/work-fleet/src/test-support/boardReadStory.ts +++ b/packages/work-fleet/src/test-support/boardReadStory.ts @@ -2,6 +2,7 @@ import { join } from "node:path"; import { NodeServices } from "@effect/platform-node"; import { Context, Deferred, Effect, Fiber, FileSystem, Layer, type Path } from "effect"; import { storyKit } from "@shivaedev/test-story/server/storyKit.ts"; +import type { Snapshot as BoardSnapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; import { Workspace } from "@shivaedev/work-board/server/workspace.ts"; import { briefRevision } from "#board/briefRevision.ts"; import { publishAfterBoardPass, withBoardReadPass, workMetadata } from "#board/readPass.ts"; @@ -27,14 +28,19 @@ export const boardReadStory = storyKit({ } const context = yield* Layer.build(state.fixture.board.pipe(Layer.provideMerge(state.fixture.services))); state.board = Context.get(context, BoardGateway); + state.workspace = Context.get(context, Workspace); state.fixture.services = Layer.succeedContext(context); - yield* state.fixture.observeWith(Context.get(context, Workspace)); + yield* state.fixture.observeWith(state.workspace); }, }, clock: "live", *create() { const fixture = yield* boardReads(0); - const optional: { board: typeof BoardGateway.Service | undefined } = { board: undefined }; + const optional: { + board: typeof BoardGateway.Service | undefined; + indexRead: BoardSnapshot | undefined; + workspace: typeof Workspace.Service | undefined; + } = { board: undefined, indexRead: undefined, workspace: undefined }; const publications: string[] = []; return { ...optional, fixture, fs: yield* FileSystem.FileSystem, publications }; }, @@ -60,6 +66,12 @@ export const boardReadStory = storyKit({ const approvedIds = approvedWorkIds(resolveHost(fixture.root, { database: syntheticDatabase, provider: "local" })).pipe( Effect.provide(fixture.services), ); + function index() { + if (state.workspace === undefined) { + throw new Error("The headless Board workspace did not open"); + } + return state.workspace.index().pipe(Effect.orDie); + } function read(id: string, options?: { readonly fresh?: boolean }) { return Effect.suspend(() => { story.tell(`read Board work ${id}${options?.fresh ? " freshly" : ""}`); @@ -86,6 +98,13 @@ export const boardReadStory = storyKit({ story.tell("defer the generated status publication"); return publishAfterBoardPass(publish); }, + directoriesReadDuring: (effect: Effect.Effect) => + Effect.gen(function* () { + const before = fixture.directoryReads(); + yield* effect; + story.tell("count the directories read meanwhile"); + return fixture.directoryReads() - before; + }), duringOnePass: (effect: Effect.Effect) => { story.tell("start a Board read pass"); return withBoardReadPass(effect).pipe(Effect.provide(fixture.services)); @@ -105,6 +124,11 @@ export const boardReadStory = storyKit({ }, }; }), + laterIndexRead: () => + Effect.map(index(), (snapshot) => { + story.tell("read the Board index again and compare it with the earlier read"); + return snapshot === state.indexRead ? "the same snapshot" : "a rebuilt snapshot"; + }), listingFailure: () => refusal(boardSnapshot(fixture.root, fixture.root).pipe(Effect.provide(fixture.services))), makeReadable: () => { story.tell("every Board file becomes readable again"); @@ -122,6 +146,11 @@ export const boardReadStory = storyKit({ return (yield* fs.exists(path)) ? yield* fs.readFileString(path) : undefined; }), read, + readIndex: () => + Effect.map(index(), (snapshot) => { + story.tell("read the Board index"); + state.indexRead = snapshot; + }), readInPass: (id: string, options?: { readonly fresh?: boolean }) => { story.tell(`read Board work ${id} in a new pass`); return withBoardReadPass(board.get(id, options)).pipe(Effect.provide(fixture.services)); @@ -172,3 +201,9 @@ export function hasBoardItems(count: number) { } }); } + +export function namesMissingArea() { + return boardReadStory.trait("files", "an open Board item names an area folder that does not exist", function* ({ fixture }) { + yield* fixture.write("area.md", "---\nid: item.area\nkind: task\nareas: [missing]\n---\n# Area\n"); + }); +} diff --git a/packages/work-fleet/src/test-support/boardReads.ts b/packages/work-fleet/src/test-support/boardReads.ts index a94221c7..8c49e336 100644 --- a/packages/work-fleet/src/test-support/boardReads.ts +++ b/packages/work-fleet/src/test-support/boardReads.ts @@ -12,6 +12,7 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu const unreadable = new Set(); let workspace: typeof Workspace.Service | undefined; let announce = true; + let directoryReads = 0; function refresh(files: readonly string[]) { return workspace && announce ? workspace.settings().pipe(Effect.flatMap(({ changes }) => changes.refreshPaths?.(files) ?? Effect.void)) @@ -21,9 +22,12 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu const tracked: FileSystem.FileSystem = { ...fs, readDirectory: (folder) => - unreadable.has(path.relative(root, folder) || ".") - ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readDirectory", module: "FileSystem" })) - : fs.readDirectory(folder), + Effect.suspend(() => { + directoryReads += 1; + return unreadable.has(path.relative(root, folder) || ".") + ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readDirectory", module: "FileSystem" })) + : fs.readDirectory(folder); + }), readFile: (file) => unreadable.has(path.relative(root, file)) ? Effect.fail(PlatformError.systemError({ _tag: "PermissionDenied", method: "readFile", module: "FileSystem" })) @@ -62,6 +66,7 @@ export const boardReads = Effect.fn("BoardTest.boardReads")(function* (count: nu Layer.provide(services), Layer.provide(Layer.succeed(Clock.Clock, Clock.Clock.defaultValue())), ), + directoryReads: () => directoryReads, observeWith: (board: typeof Workspace.Service) => Effect.sync(() => { workspace = board;