From 29091974d10cc977869f9b8675db3870943219e7 Mon Sep 17 00:00:00 2001 From: Marvin Bitterlich <644950+marvin-bitterlich@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:02:25 +0000 Subject: [PATCH 1/2] Fleet leaves settled done work asleep until its Board revision moves Done and released records stayed in the scheduler's active set while they had a Board completion action, so every scheduling write woke each of them and closeFinishedDecisions scanned the whole Board per record. Completion now stores the Board revision it last saw for the work (the brief's source and its published decisions). The scheduler skips records whose stored revision matches the current Board, and decision closure for one work reads a per-snapshot index instead of every document. --- .changeset/settled-completions.md | 8 +++ .../work-fleet/src/board/GatewayService.ts | 1 + .../src/board/closeFinishedDecisions.ts | 46 +++++++------ .../work-fleet/src/board/markdownGateway.ts | 23 ++++++- packages/work-fleet/src/board/workIndex.ts | 66 +++++++++++++++++++ .../src/engine/completeBoardWork.ts | 46 +++++++++++-- .../work-fleet/src/engine/runScheduler.ts | 8 ++- packages/work-fleet/src/engine/scheduler.ts | 8 ++- .../src/engine/settledCompletions.ts | 25 +++++++ packages/work-fleet/src/fleet.ts | 2 + packages/work-fleet/src/host/commands.ts | 37 ++++++----- .../src/host/settledCompletion.spec.ts | 63 ++++++++++++++++++ packages/work-fleet/src/model.ts | 1 + .../src/storage/boardSettledRevisions.ts | 10 +++ .../work-fleet/src/storage/fleetMigrations.ts | 2 + .../src/test-support/idleScheduler.ts | 19 ++++-- .../src/test-support/settledWorkVerbs.ts | 39 +++++++++++ 17 files changed, 350 insertions(+), 54 deletions(-) create mode 100644 .changeset/settled-completions.md create mode 100644 packages/work-fleet/src/board/workIndex.ts create mode 100644 packages/work-fleet/src/engine/settledCompletions.ts create mode 100644 packages/work-fleet/src/host/settledCompletion.spec.ts create mode 100644 packages/work-fleet/src/storage/boardSettledRevisions.ts create mode 100644 packages/work-fleet/src/test-support/settledWorkVerbs.ts diff --git a/.changeset/settled-completions.md b/.changeset/settled-completions.md new file mode 100644 index 00000000..24ba534b --- /dev/null +++ b/.changeset/settled-completions.md @@ -0,0 +1,8 @@ +--- +"@shivaedev/work-fleet": patch +--- + +### Fixed + +- `@shivaedev/work-fleet`: Done and released records whose Board completion is already applied no longer wake on every scheduling write. Fleet stores the Board revision its completion last saw (migration `38_board_settled_revision`) and reconciles such a record again only when its brief or one of its published decisions changes. +- `@shivaedev/work-fleet`: Closing finished decisions for one work item reads only that item's published decisions instead of scanning every Board document. diff --git a/packages/work-fleet/src/board/GatewayService.ts b/packages/work-fleet/src/board/GatewayService.ts index b0b1bfab..c650bd61 100644 --- a/packages/work-fleet/src/board/GatewayService.ts +++ b/packages/work-fleet/src/board/GatewayService.ts @@ -25,4 +25,5 @@ export interface BoardGatewayService { text?: string, ) => Effect.Effect; readonly sourceCreatedAt?: (sourcePath: string) => Effect.Effect; + readonly workRevisions?: () => Effect.Effect<(workId: string) => string | undefined, BoardFailure>; } diff --git a/packages/work-fleet/src/board/closeFinishedDecisions.ts b/packages/work-fleet/src/board/closeFinishedDecisions.ts index 40e6fe9b..1f09672d 100644 --- a/packages/work-fleet/src/board/closeFinishedDecisions.ts +++ b/packages/work-fleet/src/board/closeFinishedDecisions.ts @@ -4,7 +4,7 @@ import { managedRequestReceipt } from "@shivaedev/work-board/server/attention/ma import type { Snapshot as BoardSnapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; import { closePublishedDecision, publishedDecision } from "#board/closePublishedDecision.ts"; import type { ClosedDecision } from "#board/schema.ts"; -import { snapshotProblems } from "#board/workPolicy.ts"; +import { type PublishedDecisionDocument, type WorkIndex, workIndex } from "#board/workIndex.ts"; import type { BoardFailure } from "#policy.ts"; // Each closure syncs its file and directory, so a sweep in sequence waits on every sync in turn. @@ -19,16 +19,17 @@ export interface CloseFinishedOptions { export function closeFinishedDecisions(realRoot: string, snapshot: Effect.Effect) { return Effect.fn("FleetBoard.closeFinishedDecisions")(function* (options: CloseFinishedOptions = {}) { const current = yield* snapshot; - const questions = new Map( - current.documents.flatMap(({ parsed }) => - parsed.fields.question && parsed.fields.id ? [[parsed.fields.id, parsed.fields.question.source] as const] : [], - ), - ); - const answered = new Set( - current.documents.flatMap(({ parsed }) => (parsed.fields.response?.type === "answer" ? [questions.get(parsed.fields.response.question)] : [])), - ); - const candidates = current.documents.flatMap((document) => { - const candidate = finishedDecision(current, document, options, answered); + const index = workIndex(current); + const published = + options.workId === undefined + ? current.documents.flatMap((document) => { + const decision = publishedDecision(document); + return decision === undefined ? [] : [{ decision, document }]; + }) + : index.decisions(options.workId); + const answered = options.workId === undefined ? answeredQuestions(current) : new Set(); + const candidates = published.flatMap(({ decision, document }) => { + const candidate = finishedDecision(current, index, decision, document, options, answered); return candidate === undefined ? [] : [{ decision: { ...candidate, sourcePath: document.file } satisfies ClosedDecision, document }]; }); if (options.dryRun) { @@ -44,19 +45,26 @@ export function closeFinishedDecisions(realRoot: string, snapshot: Effect.Effect }); } +function answeredQuestions(snapshot: BoardSnapshot) { + const questions = new Map( + snapshot.documents.flatMap(({ parsed }) => + parsed.fields.question && parsed.fields.id ? [[parsed.fields.id, parsed.fields.question.source] as const] : [], + ), + ); + return new Set( + snapshot.documents.flatMap(({ parsed }) => (parsed.fields.response?.type === "answer" ? [questions.get(parsed.fields.response.question)] : [])), + ); +} + function finishedDecision( snapshot: BoardSnapshot, + index: WorkIndex, + decision: PublishedDecisionDocument["decision"], document: BoardDocument, options: CloseFinishedOptions, answered: ReadonlySet, ) { - const decision = publishedDecision(document); - if ( - decision === undefined - || decision.request.state !== "open" - || snapshotProblems(snapshot).has(document.file) - || (options.workId !== undefined && decision.workId !== options.workId) - ) { + if (decision.request.state !== "open" || index.problems.has(document.file)) { return; } const receipt = managedRequestReceipt(document, decision.request, snapshot.model); @@ -73,7 +81,7 @@ function finishedDecision( const matches = snapshot.model.ids.get(decision.workId); const work = matches?.length === 1 ? matches[0] : undefined; const status = work?.parsed.fields.status; - if (work === undefined || snapshotProblems(snapshot).has(work.file) || !["done", "deferred", "dropped"].includes(status ?? "")) { + if (work === undefined || index.problems.has(work.file) || !["done", "deferred", "dropped"].includes(status ?? "")) { return; } return { reason: `Closed because Board work ${decision.workId} is ${status}.`, workId: decision.workId }; diff --git a/packages/work-fleet/src/board/markdownGateway.ts b/packages/work-fleet/src/board/markdownGateway.ts index 8bc3fa40..13ff1ec8 100644 --- a/packages/work-fleet/src/board/markdownGateway.ts +++ b/packages/work-fleet/src/board/markdownGateway.ts @@ -13,6 +13,7 @@ import { workMetadata } from "#board/readPass.ts"; import { readWork } from "#board/readWork.ts"; import { recordDecisionResponse } from "#board/recordDecisionResponse.ts"; import { upgradeDecisionClosure } from "#board/upgradeDecisionClosure.ts"; +import { workIndex } from "#board/workIndex.ts"; import { metadataProblem } from "#board/workPolicy.ts"; import { workRevision } from "#board/workRevision.ts"; import { boardSnapshot } from "#board/workspace.ts"; @@ -68,9 +69,26 @@ export const MarkdownBoard = defineService({ Effect.mapError((error) => new BoardFailure({ message: error.message })), ); } - return { decisionSnapshot, documents, fs, get, path, publish, realRoot, root, snapshot, sourceCreatedAt, sources, workspace }; + function workRevisions() { + return sources.pipe(Effect.map((current) => workIndex(current).revision)); + } + return { decisionSnapshot, documents, fs, get, path, publish, realRoot, root, snapshot, sourceCreatedAt, sources, workRevisions, workspace }; }), - methods: ({ root, realRoot, decisionSnapshot, documents, get, publish, path, snapshot, sources, sourceCreatedAt, workspace, fs }) => ({ + methods: ({ + root, + realRoot, + decisionSnapshot, + documents, + get, + publish, + path, + snapshot, + sources, + sourceCreatedAt, + workRevisions, + workspace, + fs, + }) => ({ ...BoardGateway.of({ acknowledgeDecision: (input) => acknowledgeDecision( @@ -123,6 +141,7 @@ export const MarkdownBoard = defineService({ report: publishReport(root, realRoot, documents), respond: recordDecisionResponse(root, realRoot, snapshot), sourceCreatedAt, + workRevisions, }), }), requires: [BoardRoot, FileSystem.FileSystem, Path.Path, Workspace, BoardIndex], diff --git a/packages/work-fleet/src/board/workIndex.ts b/packages/work-fleet/src/board/workIndex.ts new file mode 100644 index 00000000..407072fb --- /dev/null +++ b/packages/work-fleet/src/board/workIndex.ts @@ -0,0 +1,66 @@ +import type { MetadataDocument as BoardDocument } from "@shivaedev/work-board/integration/metadata.ts"; +import { responses } from "@shivaedev/work-board/server/integration/responses.ts"; +import { documentSource } from "@shivaedev/work-board/server/metadata/model.ts"; +import type { Snapshot as BoardSnapshot } from "@shivaedev/work-board/server/search/snapshot.ts"; +import { publishedDecision } from "#board/closePublishedDecision.ts"; +import { snapshotProblems } from "#board/workPolicy.ts"; +import type { BoardFailure } from "#policy.ts"; + +const { revisionOf } = responses; + +export interface PublishedDecisionDocument { + readonly decision: NonNullable>; + readonly document: BoardDocument; +} + +export interface WorkIndex { + readonly decisions: (workId: string) => readonly PublishedDecisionDocument[]; + readonly problems: ReadonlyMap; + readonly revision: (workId: string) => string | undefined; +} + +// Snapshots never change, and the scheduler and decision closure consult them per record, so each snapshot is indexed once. +const indexes = new WeakMap(); + +export function workIndex(snapshot: BoardSnapshot): WorkIndex { + const known = indexes.get(snapshot); + if (known !== undefined) { + return known; + } + const decisions = new Map(); + for (const document of snapshot.documents) { + const decision = publishedDecision(document); + if (decision !== undefined) { + decisions.set(decision.workId, [...(decisions.get(decision.workId) ?? []), { decision, document }]); + } + } + const revisions = new Map(); + const index: WorkIndex = { + decisions: (workId) => decisions.get(workId) ?? [], + problems: snapshotProblems(snapshot), + revision: (workId) => { + if (!revisions.has(workId)) { + revisions.set(workId, workRevision(snapshot, workId, decisions.get(workId) ?? [])); + } + return revisions.get(workId); + }, + }; + indexes.set(snapshot, index); + return index; +} + +function workRevision(snapshot: BoardSnapshot, workId: string, decisions: readonly PublishedDecisionDocument[]) { + const matches = snapshot.model.ids.get(workId); + const brief = matches?.length === 1 ? matches[0] : undefined; + if (brief === undefined && decisions.length === 0) { + return; + } + return revisionOf( + JSON.stringify([ + brief === undefined ? null : [brief.file, documentSource(brief)], + ...decisions + .map(({ document }) => [document.file, documentSource(document)]) + .sort(([left], [right]) => String(left).localeCompare(String(right))), + ]), + ); +} diff --git a/packages/work-fleet/src/engine/completeBoardWork.ts b/packages/work-fleet/src/engine/completeBoardWork.ts index d2f8a4c2..3f3c6340 100644 --- a/packages/work-fleet/src/engine/completeBoardWork.ts +++ b/packages/work-fleet/src/engine/completeBoardWork.ts @@ -1,10 +1,14 @@ import { Effect } from "effect"; +import { responses } from "@shivaedev/work-board/server/integration/responses.ts"; import { briefChanged } from "#engine/boardBrief.ts"; import { refuseDeliveredId } from "#engine/deliveredId.ts"; import type { EngineState } from "#engine/makeState.ts"; import type { FleetRecord } from "#model.ts"; +import { BoardFailure } from "#policy.ts"; import type { Versioned } from "#storage/model.ts"; +const { revisionOf } = responses; + export function boardCompletionAction(record: FleetRecord) { if (record.step === "done" && record.report !== undefined) { return "Done; report recorded."; @@ -22,12 +26,22 @@ export function boardCompletionAction(record: FleetRecord) { return number === undefined ? undefined : `Done; PR #${number} merged.`; } +export function completionSettlement(action: string, revision: string | undefined) { + return revisionOf(JSON.stringify([action, revision ?? null])); +} + export function completeBoardWork(state: EngineState) { const reported = state.completionWarnings; return Effect.fn("Fleet.completeBoardWork")(function* (stored: Versioned) { const action = boardCompletionAction(stored.value); const complete = state.board.complete; const sourcePath = stored.value.boardSourcePath; + let observed = true; + function unobserved(error: BoardFailure) { + if (error.code === "incomplete-observation") { + observed = false; + } + } if ( action !== undefined && sourcePath !== undefined @@ -44,6 +58,9 @@ export function completeBoardWork(state: EngineState) { }), Effect.tap(() => Effect.sync(() => reported.delete(stored.value.workId))), Effect.catch((error) => { + if (error instanceof BoardFailure) { + unobserved(error); + } const warning = JSON.stringify([source, error.message]); if (reported.get(stored.value.workId) === warning) { return Effect.void; @@ -54,10 +71,31 @@ export function completeBoardWork(state: EngineState) { ); } if (state.board.closeFinishedDecisions !== undefined) { - yield* state.board - .closeFinishedDecisions({ workId: stored.value.workId }) - .pipe(Effect.catchTag("BoardFailure", (error) => Effect.logWarning(error.message))); + yield* state.board.closeFinishedDecisions({ workId: stored.value.workId }).pipe( + Effect.catchTag("BoardFailure", (error) => { + unobserved(error); + return Effect.logWarning(error.message); + }), + ); } - return stored; + return action === undefined || !observed ? stored : yield* settleCompletion(state, stored, action); }); } + +// Marking the Board revision that completion and decision closure last saw lets the scheduler skip this record until that revision moves. +const settleCompletion = Effect.fn("Fleet.settleCompletion")(function* (state: EngineState, stored: Versioned, action: string) { + if (state.board.workRevisions === undefined) { + return stored; + } + const revision = yield* state.board.workRevisions().pipe(Effect.option); + if (revision._tag === "None") { + return stored; + } + const settled = completionSettlement(action, revision.value(stored.value.workId)); + if (stored.value.boardSettledRevision === settled) { + return stored; + } + return yield* state + .save(stored, { ...stored.value, boardSettledRevision: settled }) + .pipe(Effect.catchTag("StorageConflict", () => Effect.succeed(stored))); +}); diff --git a/packages/work-fleet/src/engine/runScheduler.ts b/packages/work-fleet/src/engine/runScheduler.ts index bc15c4a7..9beaa3b3 100644 --- a/packages/work-fleet/src/engine/runScheduler.ts +++ b/packages/work-fleet/src/engine/runScheduler.ts @@ -29,7 +29,13 @@ export const startScheduler = Effect.fn("Fleet.startScheduler")(function* value.workId, keys, - read: fleet.list().pipe(Effect.map((rows) => activeRecords(rows).filter(({ value }) => !openSweep(value)))), + read: fleet + .list() + .pipe( + Effect.flatMap((rows) => + fleet.settledCompletions(rows).pipe(Effect.map((settled) => activeRecords(rows, settled).filter(({ value }) => !openSweep(value)))), + ), + ), }); const admission = yield* reconcile({ act: () => diff --git a/packages/work-fleet/src/engine/scheduler.ts b/packages/work-fleet/src/engine/scheduler.ts index 7ffdb595..9eccb81f 100644 --- a/packages/work-fleet/src/engine/scheduler.ts +++ b/packages/work-fleet/src/engine/scheduler.ts @@ -41,7 +41,7 @@ export const dispatchRecords = Effect.fn("Fleet.dispatchRecords")(function* (rec return dispatched; }); -export function activeRecords(records: readonly Versioned[]) { +export function activeRecords(records: readonly Versioned[], settled: ReadonlySet = new Set()) { return records .filter( ({ value }) => @@ -49,7 +49,7 @@ export function activeRecords(records: readonly Versioned[]) { || !["done", "released", "queued"].includes(value.step) || needsReleasedDelivery(value) || needsReleasedSettlement(value) - || boardCompletionAction(value) !== undefined + || (boardCompletionAction(value) !== undefined && !settled.has(value.workId)) || pendingAcknowledgements(value).length > 0 || pendingSessionCleanup(value).length > 0, ) @@ -74,7 +74,9 @@ export const cycle = inBoardReadPass( Effect.gen(function* () { const fleet = yield* Fleet; const records = yield* fleet.list(); - const active = activeRecords(records).filter(({ value }) => value.step !== "queued" || needsReleasedDelivery(value)); + const active = activeRecords(records, yield* fleet.settledCompletions(records)).filter( + ({ value }) => value.step !== "queued" || needsReleasedDelivery(value), + ); const sweeps = yield* Effect.forEach( active.filter(({ value }) => openSweep(value)), ({ value }) => fleet.reconcile(value.workId).pipe(Effect.result), diff --git a/packages/work-fleet/src/engine/settledCompletions.ts b/packages/work-fleet/src/engine/settledCompletions.ts new file mode 100644 index 00000000..8890eb29 --- /dev/null +++ b/packages/work-fleet/src/engine/settledCompletions.ts @@ -0,0 +1,25 @@ +import { Effect } from "effect"; +import { inBoardReadPass } from "#board/readPass.ts"; +import { boardCompletionAction, completionSettlement } from "#engine/completeBoardWork.ts"; +import type { FleetRecord } from "#model.ts"; +import { BoardGateway } from "#policy.ts"; +import type { Versioned } from "#storage/model.ts"; + +export const settledCompletions = Effect.fn("Fleet.settledCompletions")(function* (records: readonly Versioned[]) { + const revisions = (yield* BoardGateway).workRevisions; + const settled = new Set(); + if (revisions === undefined) { + return settled; + } + const revision = yield* inBoardReadPass(revisions()).pipe(Effect.orElseSucceed(() => undefined)); + if (revision === undefined) { + return settled; + } + for (const { value } of records) { + const action = boardCompletionAction(value); + if (action !== undefined && value.boardSettledRevision === completionSettlement(action, revision(value.workId))) { + settled.add(value.workId); + } + } + return settled; +}); diff --git a/packages/work-fleet/src/fleet.ts b/packages/work-fleet/src/fleet.ts index 36fe1bc1..7af0664f 100644 --- a/packages/work-fleet/src/fleet.ts +++ b/packages/work-fleet/src/fleet.ts @@ -20,6 +20,7 @@ import { releaseClosedCommand } from "#engine/releaseClosedCommand.ts"; import { FleetRepository } from "#engine/repository.ts"; import { resolveCommand } from "#engine/resolveCommand.ts"; import { cleanupSessions } from "#engine/sessionCleanup.ts"; +import { settledCompletions } from "#engine/settledCompletions.ts"; import { stopCommand } from "#engine/stopCommand.ts"; import { BoardGateway, FleetIntegrations, FleetPolicy } from "#policy.ts"; import type { Preparation } from "#preparation/schema.ts"; @@ -66,6 +67,7 @@ export const Fleet = defineService({ ), resolve: (workId: string, decisionId: string, responseId: string) => resolveCommand(state)(workId, decisionId, responseId).pipe(Effect.flatMap(cleanupSessions(state))), + settledCompletions, stop: stopCommand(state), }), requires: [FleetRepository, BoardGateway, FleetPolicy, FleetIntegrations, SessionService], diff --git a/packages/work-fleet/src/host/commands.ts b/packages/work-fleet/src/host/commands.ts index 1997dcc9..f0233e0e 100644 --- a/packages/work-fleet/src/host/commands.ts +++ b/packages/work-fleet/src/host/commands.ts @@ -84,22 +84,23 @@ export function hostScheduler(host: FleetHost) { }), ), ); - return Effect.gen(function* () { - const { index } = yield* BoardIndex; - const events = yield* PubSub.subscribe(index.changes.events); - const reactivity = yield* Reactivity.Reactivity; - const live = yield* LiveConfig; - const invalidate = reactivity.invalidate(invalidationKeys([schedulerInputs.list])); - yield* live.changes.pipe( - Stream.drop(1), - Stream.runForEach(() => invalidate), - Effect.forkScoped, - ); - yield* Stream.fromSubscription(events).pipe( - Stream.filter((change) => change._tag === "Watching" || change.paths.some((path) => path !== "fleet/status.md")), - Stream.runForEach(() => invalidate), - Effect.forkScoped, - ); - return yield* run(schedulerCadence, admission); - }); + return Effect.andThen(wakeSchedulerOnInputs, run(schedulerCadence, admission)); } + +export const wakeSchedulerOnInputs = Effect.gen(function* () { + const { index } = yield* BoardIndex; + const events = yield* PubSub.subscribe(index.changes.events); + const reactivity = yield* Reactivity.Reactivity; + const live = yield* LiveConfig; + const invalidate = reactivity.invalidate(invalidationKeys([schedulerInputs.list])); + yield* live.changes.pipe( + Stream.drop(1), + Stream.runForEach(() => invalidate), + Effect.forkScoped, + ); + yield* Stream.fromSubscription(events).pipe( + Stream.filter((change) => change._tag === "Watching" || change.paths.some((path) => path !== "fleet/status.md")), + Stream.runForEach(() => invalidate), + Effect.forkScoped, + ); +}); diff --git a/packages/work-fleet/src/host/settledCompletion.spec.ts b/packages/work-fleet/src/host/settledCompletion.spec.ts new file mode 100644 index 00000000..3af502c4 --- /dev/null +++ b/packages/work-fleet/src/host/settledCompletion.spec.ts @@ -0,0 +1,63 @@ +import { expect } from "@effect/vitest"; +import { approvedWork } from "#test/hostStory.ts"; +import { idleHost } from "#test/idleScheduler.ts"; +import { pendingNoChangeReview } from "#test/noChangeCompletion.ts"; + +const SUMMARY = "next_action: Done; no change needed."; + +idleHost.it( + "launching other work does not wake a done record whose completion is settled", + [approvedWork(["one"]), pendingNoChangeReview], + function* (daemon) { + yield* daemon.finishWithoutChange("one"); + const scheduler = yield* daemon.startIdleScheduler(); + const started = scheduler.reconciledWork().length; + + yield* daemon.dispatchOther("two"); + + expect(scheduler.reconciledWork().slice(started)).not.toContain("one"); + }, +); + +idleHost.it( + "an edit to a done brief wakes its record once, and Fleet applies completion again for the new revision", + [approvedWork(["one"]), pendingNoChangeReview], + function* (daemon) { + yield* daemon.finishWithoutChange("one"); + const scheduler = yield* daemon.startIdleScheduler(); + const started = scheduler.reconciledWork().length; + const done = daemon.readBoard("one.md"); + daemon.writeBoard("one.md", done.replace(SUMMARY, "next_action: Check the follow-up")); + + yield* daemon.waitForBoard("one.md", (source) => source.includes(SUMMARY)); + yield* daemon.dispatchOther("two"); + + expect( + scheduler + .reconciledWork() + .slice(started) + .filter((workId) => workId === "one"), + ).toEqual(["one"]); + }, +); + +idleHost.it( + "a done brief reopened outside Fleet is marked done again after one reconcile", + [approvedWork(["one"]), pendingNoChangeReview], + function* (daemon) { + yield* daemon.finishWithoutChange("one"); + const scheduler = yield* daemon.startIdleScheduler(); + const started = scheduler.reconciledWork().length; + daemon.writeBoard("one.md", daemon.readBoard("one.md").replace(`${SUMMARY}\n`, "").replace("status: done", "status: ready")); + + yield* daemon.waitForBoard("one.md", (source) => source.includes("status: done")); + yield* daemon.dispatchOther("two"); + + expect( + scheduler + .reconciledWork() + .slice(started) + .filter((workId) => workId === "one"), + ).toEqual(["one"]); + }, +); diff --git a/packages/work-fleet/src/model.ts b/packages/work-fleet/src/model.ts index 7d0f7808..a7fd3c5f 100644 --- a/packages/work-fleet/src/model.ts +++ b/packages/work-fleet/src/model.ts @@ -57,6 +57,7 @@ const cycleFields = { blocker: Schema.optional(Schema.String), boardContext: Schema.optional(Schema.String), boardRevision: Schema.String, + boardSettledRevision: Schema.optional(Schema.String), boardSourcePath: Schema.optional(Schema.String), checks: Schema.optional(Checks), completedAt: Schema.optional(diagnostics.FleetTimestamp), diff --git a/packages/work-fleet/src/storage/boardSettledRevisions.ts b/packages/work-fleet/src/storage/boardSettledRevisions.ts new file mode 100644 index 00000000..0a1e24ff --- /dev/null +++ b/packages/work-fleet/src/storage/boardSettledRevisions.ts @@ -0,0 +1,10 @@ +import { Effect } from "effect"; +import { SqlClient } from "effect/sql"; +import { childTable } from "#storage/recordChildren.ts"; + +export const boardSettledRevisions = Effect.fn("FleetStore.boardSettledRevisions")(function* (namespace: string) { + const sql = yield* SqlClient.SqlClient; + for (const table of [`${namespace}_records`, childTable(namespace, ["history"])]) { + yield* sql`alter table ${sql(table)} add column "boardSettledRevision" text`; + } +}); diff --git a/packages/work-fleet/src/storage/fleetMigrations.ts b/packages/work-fleet/src/storage/fleetMigrations.ts index 45261d5f..9b942c9e 100644 --- a/packages/work-fleet/src/storage/fleetMigrations.ts +++ b/packages/work-fleet/src/storage/fleetMigrations.ts @@ -3,6 +3,7 @@ import { Migrator, SqlClient } from "effect/sql"; import { addShapeColumns } from "#storage/addShapeColumns.ts"; import { advisoryReviews } from "#storage/advisoryReviews.ts"; import { backfillDecisionCodes } from "#storage/backfillDecisionCodes.ts"; +import { boardSettledRevisions } from "#storage/boardSettledRevisions.ts"; import { ciHolds } from "#storage/ciHolds.ts"; import { mergedDeliveries } from "#storage/mergedDeliveries.ts"; import { modelTables } from "#storage/modelTables.ts"; @@ -142,5 +143,6 @@ export const fleetMigrations = Effect.fn("FleetStore.fleetMigrations")(function* "attentionOwner" = coalesce("decision__attentionOwner", 'orchestrator'), version = version + 1 where step = 'reviewing' and "decision__code" = 'pr-conflicting'`, "37_ci_holds": ciHolds(namespace), + "38_board_settled_revision": boardSettledRevisions(namespace), }); }); diff --git a/packages/work-fleet/src/test-support/idleScheduler.ts b/packages/work-fleet/src/test-support/idleScheduler.ts index 015aa3d8..40160f7b 100644 --- a/packages/work-fleet/src/test-support/idleScheduler.ts +++ b/packages/work-fleet/src/test-support/idleScheduler.ts @@ -6,10 +6,11 @@ import { storyKit } from "@shivaedev/test-story/server/storyKit.ts"; import { SchedulerPasses } from "#engine/health.ts"; import { startScheduler } from "#engine/runScheduler.ts"; import { admitApproved } from "#host/admission.ts"; -import { schedulerCadence } from "#host/commands.ts"; +import { schedulerCadence, wakeSchedulerOnInputs } from "#host/commands.ts"; import type { HostStoryEngine } from "#test/HostStoryEngine.ts"; import { hostDefinition } from "#test/hostStory.ts"; import { hostVerbs } from "#test/hostVerbs.ts"; +import { settledWorkVerbs } from "#test/settledWorkVerbs.ts"; function nextPass(passes: SubscriptionRef.SubscriptionRef, after: number) { return SubscriptionRef.changes(passes).pipe( @@ -23,14 +24,14 @@ export function idleSchedulerVerbs(engine: HostStoryEngine, tell: (line: string) return { startIdleScheduler: () => Effect.gen(function* () { - tell("start Fleet admission and reconciliation on the test clock"); + tell("start Fleet admission and reconciliation on the test clock, woken by Board changes"); const clock = yield* Clock.Clock; const calls = vi.spyOn(engine.fleet, "reconcile"); yield* Effect.addFinalizer(() => Effect.sync(() => calls.mockRestore())); - const workers = yield* startScheduler(schedulerCadence, admitApproved(engine.fixture.host).pipe(Effect.orDie, Effect.asVoid)).pipe( - Effect.provideContext(engine.running.context), - Effect.provideService(Clock.Clock, clock), - ); + const workers = yield* Effect.andThen( + wakeSchedulerOnInputs, + startScheduler(schedulerCadence, admitApproved(engine.fixture.host).pipe(Effect.orDie, Effect.asVoid)), + ).pipe(Effect.provideContext(engine.running.context), Effect.provideService(Clock.Clock, clock)); const passes = Context.get(engine.running.context, SchedulerPasses); const waiting = SubscriptionRef.changes(workers.admission.state).pipe( Stream.filter((state) => state.status === "waiting"), @@ -62,5 +63,9 @@ export const idleHost = storyKit({ ...hostDefinition, after: { configuration: (engine: HostStoryEngine) => withLive(engine.deploy()) }, clock: "test", - verbs: (engine: HostStoryEngine, story) => ({ ...hostVerbs(engine, story.tell), ...idleSchedulerVerbs(engine, story.tell) }), + verbs: (engine: HostStoryEngine, story) => ({ + ...hostVerbs(engine, story.tell), + ...idleSchedulerVerbs(engine, story.tell), + ...settledWorkVerbs(engine, story.tell), + }), }); diff --git a/packages/work-fleet/src/test-support/settledWorkVerbs.ts b/packages/work-fleet/src/test-support/settledWorkVerbs.ts new file mode 100644 index 00000000..edc3a995 --- /dev/null +++ b/packages/work-fleet/src/test-support/settledWorkVerbs.ts @@ -0,0 +1,39 @@ +import { Effect, Schedule } from "effect"; +import { withLive } from "@shivaedev/test/withLive.ts"; +import { task } from "#test/fleetHost.ts"; +import { NO_CHANGE } from "#test/githubResult.ts"; +import type { HostStoryEngine } from "#test/HostStoryEngine.ts"; +import { hostBoardVerbs } from "#test/hostBoardVerbs.ts"; +import { hostVerbs } from "#test/hostVerbs.ts"; + +export function settledWorkVerbs(engine: HostStoryEngine, tell: (line: string) => void) { + const host = hostVerbs(engine, () => undefined); + const board = hostBoardVerbs(engine, () => undefined); + return { + dispatchOther: (workId: string) => + Effect.gen(function* () { + tell(`approve ${workId} on the Board and wait until Fleet launches it`); + const source = task(workId, "approved"); + board.writeBoard(`${workId}.md`, source); + yield* board.awaitIndexedSource(`${workId}.md`, source); + return yield* host.waitForRecord(workId, (record) => record.attempts[0]?.receipt !== undefined); + }), + finishWithoutChange: (workId: string) => + Effect.gen(function* () { + tell(`${workId} is reviewed as needing no change and its brief is marked done`); + const launched = yield* host.approve(workId); + host.finishTurn(launched.value, NO_CHANGE); + host.finishTurn(yield* host.reconcile(workId)); + return yield* host.reconcile(workId); + }), + waitForBoard: (file: string, ready: (source: string) => boolean) => { + tell(`wait until ${file} on the Board is ready`); + return withLive( + Effect.sync(() => board.readBoard(file)).pipe( + Effect.repeat({ schedule: Schedule.spaced("10 millis"), until: ready }), + Effect.timeout("10 seconds"), + ), + ); + }, + }; +} From 31899c4fe7e42a23368d84cd54310acdb87b3c5a Mon Sep 17 00:00:00 2001 From: Marvin Bitterlich <644950+marvin-bitterlich@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:51:11 +0000 Subject: [PATCH 2/2] Settle only applied completions and keep their update time The marker is now saved only when completion applies to the record and neither completion nor decision closure failed, so refused and inapplicable completions keep retrying as before and old deliveries stay untouched. Saving the marker alone no longer moves updatedAt. The manual test cycle keeps the full active set, since only the event-driven scheduler can rely on Board events to wake settled records. --- .../src/engine/completeBoardWork.ts | 22 ++++++------------- packages/work-fleet/src/engine/scheduler.ts | 4 +--- .../src/storage/recordTimes.test.ts | 16 ++++++++++++++ .../work-fleet/src/storage/recordTimes.ts | 15 ++++++++++++- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/packages/work-fleet/src/engine/completeBoardWork.ts b/packages/work-fleet/src/engine/completeBoardWork.ts index 3f3c6340..1cd2bea9 100644 --- a/packages/work-fleet/src/engine/completeBoardWork.ts +++ b/packages/work-fleet/src/engine/completeBoardWork.ts @@ -4,7 +4,6 @@ import { briefChanged } from "#engine/boardBrief.ts"; import { refuseDeliveredId } from "#engine/deliveredId.ts"; import type { EngineState } from "#engine/makeState.ts"; import type { FleetRecord } from "#model.ts"; -import { BoardFailure } from "#policy.ts"; import type { Versioned } from "#storage/model.ts"; const { revisionOf } = responses; @@ -36,18 +35,13 @@ export function completeBoardWork(state: EngineState) { const action = boardCompletionAction(stored.value); const complete = state.board.complete; const sourcePath = stored.value.boardSourcePath; - let observed = true; - function unobserved(error: BoardFailure) { - if (error.code === "incomplete-observation") { - observed = false; - } - } - if ( + const applicable = action !== undefined && sourcePath !== undefined && complete !== undefined - && stored.value.attempts.some((attempt) => attempt.role !== "reviewer" && attempt.receipt !== undefined) - ) { + && stored.value.attempts.some((attempt) => attempt.role !== "reviewer" && attempt.receipt !== undefined); + let failed = false; + if (applicable) { let source = sourcePath; yield* state.board.get(stored.value.workId).pipe( Effect.flatMap((work) => { @@ -58,9 +52,7 @@ export function completeBoardWork(state: EngineState) { }), Effect.tap(() => Effect.sync(() => reported.delete(stored.value.workId))), Effect.catch((error) => { - if (error instanceof BoardFailure) { - unobserved(error); - } + failed = true; const warning = JSON.stringify([source, error.message]); if (reported.get(stored.value.workId) === warning) { return Effect.void; @@ -73,12 +65,12 @@ export function completeBoardWork(state: EngineState) { if (state.board.closeFinishedDecisions !== undefined) { yield* state.board.closeFinishedDecisions({ workId: stored.value.workId }).pipe( Effect.catchTag("BoardFailure", (error) => { - unobserved(error); + failed = true; return Effect.logWarning(error.message); }), ); } - return action === undefined || !observed ? stored : yield* settleCompletion(state, stored, action); + return applicable && !failed ? yield* settleCompletion(state, stored, action) : stored; }); } diff --git a/packages/work-fleet/src/engine/scheduler.ts b/packages/work-fleet/src/engine/scheduler.ts index 9eccb81f..1359456e 100644 --- a/packages/work-fleet/src/engine/scheduler.ts +++ b/packages/work-fleet/src/engine/scheduler.ts @@ -74,9 +74,7 @@ export const cycle = inBoardReadPass( Effect.gen(function* () { const fleet = yield* Fleet; const records = yield* fleet.list(); - const active = activeRecords(records, yield* fleet.settledCompletions(records)).filter( - ({ value }) => value.step !== "queued" || needsReleasedDelivery(value), - ); + const active = activeRecords(records).filter(({ value }) => value.step !== "queued" || needsReleasedDelivery(value)); const sweeps = yield* Effect.forEach( active.filter(({ value }) => openSweep(value)), ({ value }) => fleet.reconcile(value.workId).pipe(Effect.result), diff --git a/packages/work-fleet/src/storage/recordTimes.test.ts b/packages/work-fleet/src/storage/recordTimes.test.ts index a953e1e8..7b1f2c8e 100644 --- a/packages/work-fleet/src/storage/recordTimes.test.ts +++ b/packages/work-fleet/src/storage/recordTimes.test.ts @@ -38,6 +38,22 @@ timedStorage.it("an edit of finished work keeps its completion and step entry ti expect(yield* store.reload(recorded.value.workId)).toEqual(recorded); }); +timedStorage.it( + "recording the Board revision finished work settled on keeps its update time", + [], + function* ({ now, openFleetStorage, secondPasses }) { + const store = yield* openFleetStorage({ namespace: STORAGE_NAMESPACE }); + const start = yield* now(); + const queued = yield* store.insert(itemRecord("queued")); + const done = yield* store.replace(queued.value.workId, queued.version, { ...queued.value, step: "done" }); + yield* secondPasses(); + + const settled = yield* store.replace(done.value.workId, done.version, { ...done.value, boardSettledRevision: "settled-revision" }); + + expect(settled.value).toMatchObject({ boardSettledRevision: "settled-revision", completedAt: start, updatedAt: start }); + }, +); + timedStorage.it( "requeueing finished work after a restart enters the queue now and clears its completion time", [], diff --git a/packages/work-fleet/src/storage/recordTimes.ts b/packages/work-fleet/src/storage/recordTimes.ts index 8223970c..a488d6b5 100644 --- a/packages/work-fleet/src/storage/recordTimes.ts +++ b/packages/work-fleet/src/storage/recordTimes.ts @@ -3,8 +3,21 @@ import { fleetStage, StageInput } from "#stage.ts"; export const recordTimeColumns = { completedAt: "number", stageEnteredAt: "number", updatedAt: "number" } as const; +// The settled Board revision is scheduler bookkeeping, so recording it alone does not count as an update. +const BOOKKEEPING = new Set(["boardSettledRevision", "completedAt", "stageEnteredAt", "updatedAt"]); + +function updateContent(record: Readonly>) { + return JSON.stringify( + Object.keys(record) + .filter((key) => !BOOKKEEPING.has(key) && record[key] !== undefined) + .sort((left, right) => left.localeCompare(right)) + .map((key) => [key, record[key]]), + ); +} + export function recordTimes(record: Readonly>, previous: Readonly> | undefined, now: number) { const sameStage = previous !== undefined && previous.stage === record.stage; + const bookkeeping = previous !== undefined && sameStage && updateContent(record) === updateContent(previous); const enteredAt = sameStage ? previous.stageEnteredAt : now; let completedAt: unknown; if (record.stage === "done") { @@ -14,7 +27,7 @@ export function recordTimes(record: Readonly>, previous: ...record, completedAt, stageEnteredAt: enteredAt, - updatedAt: now, + updatedAt: bookkeeping ? previous.updatedAt : now, }; }