From 97ad98e41d5636fdee968f9996cafc0fd8e8e24b Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Tue, 17 Nov 2026 16:30:10 -0500 Subject: [PATCH] Capture failure journals for Agent, Testing and Git test owners Replay the reviewed #1363 patch onto current main. --- .../acp/tests/agent-certification.test.ts | 214 ++++++----- packages/agent/tests/agent-components.test.ts | 213 ++++++----- .../tests/agent-function-components.test.ts | 42 ++- packages/agent/tests/agent-usage.test.ts | 93 +++-- packages/git/tests/issue-markdown.test.ts | 140 ++++--- packages/git/tests/pull-request-read.test.ts | 156 ++++---- packages/test-agent/tests/components.test.ts | 353 +++++++++++------- .../testing/tests/execution-harness.test.ts | 25 +- 8 files changed, 714 insertions(+), 522 deletions(-) diff --git a/packages/acp/tests/agent-certification.test.ts b/packages/acp/tests/agent-certification.test.ts index f0d4eb71a..b089e488d 100644 --- a/packages/acp/tests/agent-certification.test.ts +++ b/packages/acp/tests/agent-certification.test.ts @@ -1,5 +1,9 @@ -import { captureTestJournal } from "@executablemd/test-support/failure-journal"; -import { describe, it } from "@executablemd/test-support/bdd"; +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; +import { beforeEach, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { ensure, scoped, spawn, withResolvers } from "effection"; import type { Operation, Stream } from "effection"; @@ -50,7 +54,7 @@ function* document( documentSource: string, report: Report, failures: unknown[] = [], - journal = new InMemoryStream(), + journal?: InMemoryStream, ): Operation<{ ok: boolean; output: string }> { return yield* scoped(function* () { const command = mode === "strong" ? "codex" : f.command(mode); @@ -109,7 +113,8 @@ function* document( yield* useNormalizedOutput(); const path = join(f.dir, "document.md"); yield* writeTextFile(path, documentSource); - const execution = yield* executeInstalled({ path, stream: journal }, [ + const stream = journal ?? (yield* useJournalStream(path)); + const execution = yield* executeInstalled({ path, stream }, [ { components: agentIdentityComponents() }, ]); const subscription = yield* execution.output; @@ -126,6 +131,7 @@ describe( "production local-agent qualification controls", { sanitizeOps: false, sanitizeResources: false }, () => { + beforeEach(journalBeforeEach("packages/acp/tests/agent-certification.test.ts")); it("recovers a private marker in sequential Prompts and keeps another Session independent", function* () { yield* withCertificationFixture(function* (f) { const marker = randomUUID(); @@ -262,109 +268,119 @@ describe( expect((yield* f.methods()).filter((method) => method === "session/prompt").length).toBe(1); }); }); - it("attributes the owned still-active turn to HostExecution.stop without certifying native cleanup", function* () { + const cancellationCase = + "attributes the owned still-active turn to HostExecution.stop without certifying native cleanup"; + it(cancellationCase, function* () { yield* withCertificationFixture(function* (f) { const command = f.command("standard-wait"); const report = reportFor(command); report.capabilities.cancellation.selected = true; let closed = 0; - yield* scoped(function* () { - yield* installAgentComponents({ - defaultAgent: command, - rootProvider: { - options: { defaultAgent: command, permissionMode: "deny-all" }, - factory: createAcpxProvider({ - sessionStore: createAcpxSessionStore(join(f.dir, "store")), - createRuntime(options) { - const runtime = createAcpRuntime(options); - const close = runtime.close.bind(runtime); - runtime.close = (input) => - close(input).then(() => { - closed++; - }); - return runtime; - }, - *agentCwd() { - return f.dir; + const path = join(f.dir, "cancel.md"); + yield* captureTestJournal( + { + owner: "packages/acp/tests/agent-certification.test.ts", + suite: "production local-agent qualification controls", + name: cancellationCase, + }, + path, + (journal) => + scoped(function* () { + yield* installAgentComponents({ + defaultAgent: command, + rootProvider: { + options: { defaultAgent: command, permissionMode: "deny-all" }, + factory: createAcpxProvider({ + sessionStore: createAcpxSessionStore(join(f.dir, "store")), + createRuntime(options) { + const runtime = createAcpRuntime(options); + const close = runtime.close.bind(runtime); + runtime.close = (input) => + close(input).then(() => { + closed++; + }); + return runtime; + }, + *agentCwd() { + return f.dir; + }, + }), }, - }), - }, - }); - const started = withResolvers(); - const waiting = withResolvers(); - let terminal = false; - yield* Agent.around({ - *prompt(args, next) { - const source = yield* next(...args); - const observed: Stream = { - *[Symbol.iterator]() { - const subscription = yield* source; - return { - *next() { - const event = yield* subscription.next(); - if (!event.done && event.value.type === "started") { - started.resolve(); - } - if ( - !event.done && - event.value.type === "text_delta" && - event.value.text === "WAITING" - ) { - waiting.resolve(); - } - if (!event.done && event.value.type === "terminal") { - terminal = true; - } - return event; + }); + const started = withResolvers(); + const waiting = withResolvers(); + let terminal = false; + yield* Agent.around({ + *prompt(args, next) { + const source = yield* next(...args); + const observed: Stream = { + *[Symbol.iterator]() { + const subscription = yield* source; + return { + *next() { + const event = yield* subscription.next(); + if (!event.done && event.value.type === "started") { + started.resolve(); + } + if ( + !event.done && + event.value.type === "text_delta" && + event.value.text === "WAITING" + ) { + waiting.resolve(); + } + if (!event.done && event.value.type === "terminal") { + terminal = true; + } + return event; + }, + }; }, }; + return observed; }, - }; - return observed; - }, - }); - const path = join(f.dir, "cancel.md"); - yield* writeTextFile( - path, - '', - ); - const journal = new InMemoryStream(); - const execution = yield* executeInstalled({ path, stream: journal }, [ - { components: agentIdentityComponents() }, - ]); - const subscription = yield* execution.output; - const draining = yield* spawn(function* () { - while (!(yield* subscription.next()).done) {} - }); - yield* started.operation; - // Transport submission can precede the peer installing its cancellable request. - yield* waiting.operation; - const activeAtStop = !terminal; - const stop = yield* execution.stop(); - const completion = yield* execution; - yield* draining; - expect(stop.ok).toBe(true); - expect(completion.ok).toBe(false); - expect(closed).toBe(1); - const events = journal.snapshot(); - const interrupted = events.some( - (event) => event.type === "close" && event.result.status === "cancelled", - ); - expect(interrupted).toBe(true); - recordCancellation(report, { - requestedStop: true, - activeAtStop, - attributed: stop.ok && interrupted, - completed: terminal, - ownerJoined: true, - nativeCleanupVerified: false, - }); - expect(report.capabilities.cancellation.observations.attributed).toBe(true); - expect(report.capabilities.cancellation.reason).toBe("cleanup-unverified"); - const methods = yield* f.methods(); - expect(methods.filter((method) => method === "session/prompt").length).toBe(1); - expect(methods.includes("session/cancel")).toBe(true); - }); + }); + yield* writeTextFile( + path, + '', + ); + const execution = yield* executeInstalled({ path, stream: journal }, [ + { components: agentIdentityComponents() }, + ]); + const subscription = yield* execution.output; + const draining = yield* spawn(function* () { + while (!(yield* subscription.next()).done) {} + }); + yield* started.operation; + // Transport submission can precede the peer installing its cancellable request. + yield* waiting.operation; + const activeAtStop = !terminal; + const stop = yield* execution.stop(); + const completion = yield* execution; + yield* draining; + expect(stop.ok).toBe(true); + expect(completion.ok).toBe(false); + expect(closed).toBe(1); + const events = journal.snapshot(); + const interrupted = events.some( + (event) => event.type === "close" && event.result.status === "cancelled", + ); + expect(interrupted).toBe(true); + recordCancellation(report, { + requestedStop: true, + activeAtStop, + attributed: stop.ok && interrupted, + completed: terminal, + ownerJoined: true, + nativeCleanupVerified: false, + }); + expect(report.capabilities.cancellation.observations.attributed).toBe(true); + expect(report.capabilities.cancellation.reason).toBe("cleanup-unverified"); + const methods = yield* f.methods(); + expect(methods.filter((method) => method === "session/prompt").length).toBe(1); + expect(methods.includes("session/cancel")).toBe(true); + }), + ); }); }); it("observes eager configuration refusal on the established Session", function* () { @@ -436,6 +452,7 @@ describe( ); describe("standard completed replay", { sanitizeOps: false, sanitizeResources: false }, () => { + beforeEach(journalBeforeEach("packages/acp/tests/agent-certification.test.ts")); it("completed standard output replays without provider requests", function* () { yield* withCertificationFixture(function* (f) { const journal = new InMemoryStream(); @@ -465,6 +482,7 @@ describe("standard completed replay", { sanitizeOps: false, sanitizeResources: f }); describe("strong qualification evidence", { sanitizeOps: false, sanitizeResources: false }, () => { + beforeEach(journalBeforeEach("packages/acp/tests/agent-certification.test.ts")); it("reports declared acceptance and asserted identity independently", function* () { yield* withCertificationFixture(function* (f) { const report = reportFor("codex"); diff --git a/packages/agent/tests/agent-components.test.ts b/packages/agent/tests/agent-components.test.ts index 831701690..5e35b130a 100644 --- a/packages/agent/tests/agent-components.test.ts +++ b/packages/agent/tests/agent-components.test.ts @@ -28,8 +28,11 @@ import type { AgentProviderFactory } from "../src/provider-api.ts"; import { installAgentComponents } from "../src/components.ts"; import { Component } from "../../core/src/component-api.ts"; import type { Json } from "@executablemd/core"; -import { journalBeforeEach, useJournalStream } from "@executablemd/test-support/failure-journal"; - +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; const OWNER = "packages/agent/tests/agent-components.test.ts"; interface StubResponse { @@ -153,7 +156,7 @@ function* installStub(stub: Stub): Operation { function* runDoc( doc: string, - providedStream?: InMemoryStream, + suppliedStream?: InMemoryStream, dir = path.join(os.tmpdir(), `xmd-ac-test-${randomUUID()}`), ): Operation<{ output: string; result: Result }> { yield* ensureDir(dir); @@ -162,7 +165,7 @@ function* runDoc( const docPath = path.join(dir, "doc.md"); yield* writeTextFile(docPath, doc); - const stream = providedStream ?? (yield* useJournalStream(docPath)); + const stream = suppliedStream ?? (yield* useJournalStream(docPath)); // `` names durable work after its own invocation, so the host // declares it to the execution rather than registering it. const execution = yield* executeInstalled({ path: docPath, stream }, [ @@ -180,6 +183,18 @@ function* runDoc( const SUITE = "Tier AC — agent components"; +function captureAgentDocument( + name: string, + body: (stream: InMemoryStream, directory: string) => Operation, +): Operation { + const dir = path.join(os.tmpdir(), `xmd-ac-test-${randomUUID()}`); + return captureTestJournal( + { owner: OWNER, suite: SUITE, name }, + path.join(dir, "doc.md"), + (stream) => body(stream, dir), + ); +} + describe(SUITE, () => { beforeEach(journalBeforeEach(OWNER)); it("AC1: wrapper always sends rendered children, untrimmed", function* () { @@ -249,34 +264,39 @@ describe(SUITE, () => { // The composition the whole component exists for, black box: a value bound by // ``, rendered by ``, consumed and retained by ``. Import // graphs cannot prove authored syntax, so the exact bytes are pinned here. - it("AC21: sends and retains exactly what rendered", function* () { - const stub = createStubProvider(); - yield* installStub(stub); - const stream = new InMemoryStream(); - const { result } = yield* runDoc( - '\n\n' + - "beforeafter\n", - stream, - ); - expect(result.ok).toBe(true); - - const expected = 'before{\n "type": "object",\n "required": [\n "bump"\n ]\n}after'; - expect(stub.promptCalls.length).toBe(1); - // Two-space JSON, the authored bytes on either side, and no newline of its - // own between them. - expect(stub.promptCalls[0]!.content).toBe(expected); - - const events = yield* stream.readAll(); - const prompts = events.filter( - (event) => event.type === "yield" && event.description.type === "agent_prompt", + { + const name = "AC21: sends and retains exactly what rendered"; + it(name, () => + captureAgentDocument(name, function* (stream, dir) { + const stub = createStubProvider(); + yield* installStub(stub); + const { result } = yield* runDoc( + '\n\n' + + "beforeafter\n", + stream, + dir, + ); + expect(result.ok).toBe(true); + + const expected = 'before{\n "type": "object",\n "required": [\n "bump"\n ]\n}after'; + expect(stub.promptCalls.length).toBe(1); + // Two-space JSON, the authored bytes on either side, and no newline of its + // own between them. + expect(stub.promptCalls[0]!.content).toBe(expected); + + const events = yield* stream.readAll(); + const prompts = events.filter( + (event) => event.type === "yield" && event.description.type === "agent_prompt", + ); + expect(prompts.length).toBe(1); + const entry = prompts[0]!; + if (entry.type === "yield") { + // `` still owns its own record, holding the text it consumed. + expect(entry.description.input).toBe(expected); + } + }), ); - expect(prompts.length).toBe(1); - const entry = prompts[0]!; - if (entry.type === "yield") { - // `` still owns its own record, holding the text it consumed. - expect(entry.description.input).toBe(expected); - } - }); + } it("AC4: as binding captures the response instead of emitting it", function* () { const stub = createStubProvider(); @@ -442,72 +462,81 @@ describe(SUITE, () => { expect(stub.promptCalls.length).toBe(2); }); - it("AC15: journal entries carry the full prompt record", function* () { - const stub = createStubProvider(() => ({ - status: "failed", - stopReason: "max_tokens", - deltas: ["partial"], - })); - yield* installStub(stub); - const stream = new InMemoryStream(); - yield* runDoc('\n', stream); - - const events = yield* stream.readAll(); - const prompts = events.filter( - (event) => event.type === "yield" && event.description.type === "agent_prompt", - ); - expect(prompts.length).toBe(1); - const entry = prompts[0]!; - if (entry.type === "yield") { - expect(entry.description.name).toMatch(/^prompt:.*#0$/); - expect(entry.description.input).toBe("describe"); - expect(entry.result.status).toBe("ok"); - if (entry.result.status === "ok") { - expect(entry.result.value).toMatchObject({ - sequence: 0, - agent: "stub-agent", - sessionKey: "stub:default", + { + const name = "AC15: journal entries carry the full prompt record"; + it(name, () => + captureAgentDocument(name, function* (stream, dir) { + const stub = createStubProvider(() => ({ status: "failed", stopReason: "max_tokens", - text: "partial", - }); - } - } - }); - - it("AC17: throwOnError failures journal a raised marker; plain failures do not", function* () { - const stub = createStubProvider((content) => - content.includes("boom") ? { status: "failed", stopReason: "refusal" } : {}, - ); - yield* installStub(stub); - const stream = new InMemoryStream(); - yield* runDoc('\n', stream); - - const events = yield* stream.readAll(); - const plain = events.find( - (event) => event.type === "yield" && event.description.type === "agent_prompt", + deltas: ["partial"], + })); + yield* installStub(stub); + yield* runDoc('\n', stream, dir); + + const events = yield* stream.readAll(); + const prompts = events.filter( + (event) => event.type === "yield" && event.description.type === "agent_prompt", + ); + expect(prompts.length).toBe(1); + const entry = prompts[0]!; + if (entry.type === "yield") { + expect(entry.description.name).toMatch(/^prompt:.*#0$/); + expect(entry.description.input).toBe("describe"); + expect(entry.result.status).toBe("ok"); + if (entry.result.status === "ok") { + expect(entry.result.value).toMatchObject({ + sequence: 0, + agent: "stub-agent", + sessionKey: "stub:default", + status: "failed", + stopReason: "max_tokens", + text: "partial", + }); + } + } + }), ); - if (plain && plain.type === "yield" && plain.result.status === "ok") { - const value = plain.result.value; - if (typeof value === "object" && value !== null && !Array.isArray(value)) { - expect(value.raised).toBe(undefined); - } - } + } + + { + const name = "AC17: throwOnError failures journal a raised marker; plain failures do not"; + it(name, () => + captureAgentDocument(name, function* (stream, dir) { + const stub = createStubProvider((content) => + content.includes("boom") ? { status: "failed", stopReason: "refusal" } : {}, + ); + yield* installStub(stub); + yield* runDoc('\n', stream, dir); + + const events = yield* stream.readAll(); + const plain = events.find( + (event) => event.type === "yield" && event.description.type === "agent_prompt", + ); + if (plain && plain.type === "yield" && plain.result.status === "ok") { + const value = plain.result.value; + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + expect(value.raised).toBe(undefined); + } + } - const thrown = new InMemoryStream(); - yield* runDoc('\n', thrown); - const thrownEvents = yield* thrown.readAll(); - const raisedEntry = thrownEvents.find( - (event) => event.type === "yield" && event.description.type === "agent_prompt", + yield* captureAgentDocument(name, function* (thrown, thrownDir) { + yield* runDoc('\n', thrown, thrownDir); + const thrownEvents = yield* thrown.readAll(); + const raisedEntry = thrownEvents.find( + (event) => event.type === "yield" && event.description.type === "agent_prompt", + ); + expect(raisedEntry).toBeDefined(); + if (raisedEntry && raisedEntry.type === "yield" && raisedEntry.result.status === "ok") { + const value = raisedEntry.result.value; + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + expect(value.raised).toBe(true); + } + } + }); + }), ); - expect(raisedEntry).toBeDefined(); - if (raisedEntry && raisedEntry.type === "yield" && raisedEntry.result.status === "ok") { - const value = raisedEntry.result.value; - if (typeof value === "object" && value !== null && !Array.isArray(value)) { - expect(value.raised).toBe(true); - } - } - }); + } it("AC18: full replay omits raised failures from aggregate restoration", function* () { const stream = new InMemoryStream(); diff --git a/packages/agent/tests/agent-function-components.test.ts b/packages/agent/tests/agent-function-components.test.ts index d3ad9c6fe..9d3336e63 100644 --- a/packages/agent/tests/agent-function-components.test.ts +++ b/packages/agent/tests/agent-function-components.test.ts @@ -375,7 +375,7 @@ interface RunOptions { function* runDoc( doc: string, options: RunOptions = {}, - sharedJournal = false, + sharedJournal = true, ): Operation<{ output: string; result: Result; trace: Trace; events: DurableEvent[] }> { const trace: Trace = options.trace ?? { prompts: [], agentLookups: [], timeouts: [] }; const dir = path.join(os.tmpdir(), `xmd-af-test-${randomUUID()}`); @@ -932,11 +932,12 @@ function promptRecords(events: DurableEvent[]): { } describe("Tier AF — Session configuration", () => { + beforeEach(journalBeforeEach(OWNER)); beforeAll(() => useTempFileCompiler()); it("AF26: the exact configuration reaches the provider, and not through the route", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result } = yield* runDoc( + const { result } = yield* run( ['', ""].join("\n"), { trace }, ); @@ -953,7 +954,7 @@ describe("Tier AF — Session configuration", () => { it("AF27: an unconfigured Session asks the provider nothing at all", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result } = yield* runDoc('\n', { trace }); + const { result } = yield* run('\n', { trace }); expect(result.ok).toBe(true); // Nothing was asked: an element that configured nothing has nothing to ask @@ -966,7 +967,7 @@ describe("Tier AF — Session configuration", () => { it("AF28: a prompt retains the one configuration its Session runs under", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( [ '', '', @@ -988,7 +989,7 @@ describe("Tier AF — Session configuration", () => { }); it("AF29: a setting nobody authored is retained as nothing at all", function* () { - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1001,7 +1002,7 @@ describe("Tier AF — Session configuration", () => { it("AF30: a prompt under no configured Session retains neither member", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc('\n', { trace }); + const { result, events } = yield* run('\n', { trace }); expect(result.ok).toBe(true); const [prompt] = promptRecords(events); @@ -1010,7 +1011,7 @@ describe("Tier AF — Session configuration", () => { it("AF31: an inner Session does not inherit the outer element's request", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( [ '', '', @@ -1035,7 +1036,7 @@ describe("Tier AF — Session configuration", () => { }); it("AF35: a prompt refused before its turn started records no configuration", function* () { - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1051,7 +1052,7 @@ describe("Tier AF — Session configuration", () => { }); it("AF36: a prompt that failed after starting retains what it ran under", function* () { - const { events } = yield* runDoc( + const { events } = yield* run( [ '', '', @@ -1160,9 +1161,10 @@ function recordedFailure(record: Record | undefined): string { * either. */ describe("Tier AF — the final routed Session decides", () => { + beforeEach(journalBeforeEach(OWNER)); it("AF37: routing the raw session a use names runs unconfigured", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1190,7 +1192,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF38: routing no session at all is an ordinary unconfigured route", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1218,7 +1220,7 @@ describe("Tier AF — the final routed Session decides", () => { // what the record says — is the first's settings, because that is the // conversation it actually ran in. let first: string | Session | undefined; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( [ '', '', @@ -1253,7 +1255,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF40: a value that claims to be configured and was issued by nobody refuses", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1284,7 +1286,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF41: reading the use and delegating it is what a handler is for", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; const seen: string[] = []; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( [ '', '', @@ -1314,7 +1316,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF42: one configured Session hands the same use to every nested prompt", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; const routed: (string | Session | undefined)[] = []; - const { result } = yield* runDoc( + const { result } = yield* run( [ '', '', @@ -1347,7 +1349,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF43: a programmatic prompt beneath the element runs in its conversation", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; const seen: boolean[] = []; - const { result } = yield* runDoc( + const { result } = yield* run( ['', "", "", ""].join("\n"), { trace, @@ -1372,7 +1374,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF44: a programmatic prompt a handler reroutes follows that route", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result } = yield* runDoc( + const { result } = yield* run( ['', "", "", ""].join("\n"), { trace, @@ -1395,7 +1397,7 @@ describe("Tier AF — the final routed Session decides", () => { it("AF45: a Session a handler answers with never went through a setup", function* () { const trace: Trace = { prompts: [], agentLookups: [], timeouts: [] }; - const { result, events } = yield* runDoc( + const { result, events } = yield* run( ['', '', "", ""].join( "\n", ), @@ -1463,6 +1465,7 @@ function mentions(value: unknown, canary: string): boolean { } describe("Tier AF — the permission audit one turn retains", () => { + beforeEach(journalBeforeEach(OWNER)); beforeAll(() => useTempFileCompiler()); it("A1: a turn that was asked nothing retains no audit at all", function* () { @@ -1793,6 +1796,7 @@ describe("Tier AF — the permission audit one turn retains", () => { }); describe("AFP — a publisher that never appends retains nothing", () => { + beforeEach(journalBeforeEach(OWNER)); beforeAll(() => useTempFileCompiler()); it("AFP0: the same harness with a publisher that appends succeeds and records", function* () { @@ -1980,6 +1984,7 @@ function diagnosticRecord(events: DurableEvent[]) { } describe("1039 saved Prompt diagnostics", () => { + beforeEach(journalBeforeEach(OWNER)); for (const raised of [false, true]) { it(`A1/A3: selected facts survive publication and ${raised ? "thrown" : "aggregate"} reconstruction`, function* () { const diagnostic = { ...DIAGNOSTIC, acp: { ...DIAGNOSTIC.acp, details: "" } }; @@ -2136,6 +2141,7 @@ describe("1039 saved Prompt diagnostics", () => { }); describe("Try Prompt normalization", () => { + beforeEach(journalBeforeEach(OWNER)); it("withholds a failed-turn record when the subscribed stream fails release", function* () { const result = yield* runDoc('', { handler: () => diagnosticPrompt(new Error("ordinary"), new Error("release failed")), diff --git a/packages/agent/tests/agent-usage.test.ts b/packages/agent/tests/agent-usage.test.ts index c2b527db5..97043bbd8 100644 --- a/packages/agent/tests/agent-usage.test.ts +++ b/packages/agent/tests/agent-usage.test.ts @@ -7,7 +7,11 @@ * path, replay without a provider, and direct calls that retain nothing. */ -import { journalBeforeEach, useJournalStream } from "@executablemd/test-support/failure-journal"; +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; import { beforeEach, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; @@ -549,50 +553,58 @@ describe("Tier PU — canonical retention and replay", () => { expect(record?.checkpoint).toBe(undefined); }); - it("PU-P3: a held conversation's turn retains usage through the same runner", function* () { - const stream = new InMemoryStream(); - const trace = newTrace(); - const factory = stubFactory(trace, always({ usage: () => versioned("held") })); - const installation: ExecutionInstallation = { - *install(): Operation { - yield* installAgentComponents({ - rootProvider: { - factory, - options: { defaultAgent: AGENT, permissionMode: "deny-all" }, + it("PU-P3: a held conversation's turn retains usage through the same runner", () => + captureTestJournal( + { + owner: OWNER, + name: "PU-P3: a held conversation's turn retains usage through the same runner", + }, + "agent/conversation.md", + function* (stream) { + const trace = newTrace(); + const factory = stubFactory(trace, always({ usage: () => versioned("held") })); + const installation: ExecutionInstallation = { + *install(): Operation { + yield* installAgentComponents({ + rootProvider: { + factory, + options: { defaultAgent: AGENT, permissionMode: "deny-all" }, + }, + }); }, + }; + const answered = yield* scoped(function* () { + const opened = yield* useAgentConversation({ + history: stream, + id: "chat-usage", + agent: AGENT, + installations: [installation], + }); + if (!opened.ok) { + throw opened.error; + } + return yield* opened.value.prompt("hello"); }); + expect(answered.ok ? "ok" : answered.error.message).toBe("ok"); + const records = promptValues(yield* stream.readAll()); + expect(records.length).toBe(1); + expect(parsePromptRecord(records[0])?.usage?.provenance.xmd.version).toBe("held"); + expect(trace.subscriptions).toBe(1); }, - }; - const answered = yield* scoped(function* () { - const opened = yield* useAgentConversation({ - history: stream, - id: "chat-usage", - agent: AGENT, - installations: [installation], - }); - if (!opened.ok) { - throw opened.error; - } - return yield* opened.value.prompt("hello"); - }); - expect(answered.ok ? "ok" : answered.error.message).toBe("ok"); - const records = promptValues(yield* stream.readAll()); - expect(records.length).toBe(1); - expect(parsePromptRecord(records[0])?.usage?.provenance.xmd.version).toBe("held"); - expect(trace.subscriptions).toBe(1); - }); + )); it("PU-P4: an unsafe model name is withheld by the producing execution; the rest is kept", function* () { const marker = ["tok", "en-", "USAGE01"].join(""); const unsafe = sampleUsage("unsafe", marker); - const { result, events } = yield* scoped(function* () { - return yield* runDocument( - "one\n", - new InMemoryStream(), - stubFactory(newTrace(), always({ usage: () => unsafe })), - { scanner: markerScanner(marker) }, - ); - }); + const directory = yield* useTempDirectory("xmd-pu-test-"); + const stream = yield* useJournalStream(path.join(directory, "doc.md")); + const { result, events } = yield* runDocument( + "one\n", + stream, + stubFactory(newTrace(), always({ usage: () => unsafe })), + { scanner: markerScanner(marker) }, + directory, + ); expect(result.ok).toBe(true); const retained = parsePromptRecord(promptValues(events)[0])?.usage; expect(JSON.stringify(retained).includes(marker)).toBe(false); @@ -623,6 +635,7 @@ function markerScanner(marker: string): () => SecretScanner { } describe("Tier PU — producing-execution disclosure", () => { + beforeEach(journalBeforeEach(OWNER)); it("PU-P5: a protected input stays withheld under scanner opt-out, whatever context is rebound", function* () { const owner = yield* createInputOwner(); const values = yield* attachInputOwner(owner); @@ -691,12 +704,14 @@ describe("Tier PU — producing-execution disclosure", () => { : []; }, }); - const stream = new InMemoryStream(); + const directory = yield* useTempDirectory("xmd-pu-test-"); + const stream = yield* useJournalStream(path.join(directory, "doc.md")); const { result, events } = yield* runDocument( "one\n", stream, stubFactory(newTrace(), always({ usage: () => sampleUsage("gated") })), { scanner }, + directory, ); expect(result.ok).toBe(false); expect(promptValues(events)).toEqual([]); diff --git a/packages/git/tests/issue-markdown.test.ts b/packages/git/tests/issue-markdown.test.ts index 8ae85a758..ac8a4cb14 100644 --- a/packages/git/tests/issue-markdown.test.ts +++ b/packages/git/tests/issue-markdown.test.ts @@ -11,7 +11,7 @@ * `github-issues.test.ts`, and nothing there duplicates a scenario here. */ -import { describe, it } from "@executablemd/test-support/bdd"; +import { beforeEach, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; @@ -20,7 +20,13 @@ import type { Operation } from "effection"; import { SOURCE_POSITION_FIELD } from "@executablemd/core"; import { issueYields, runScenario, useScenarioFixture } from "./support/issue-scenario.ts"; import type { ScenarioObservation } from "./support/issue-scenario.ts"; -import { captureTestJournal } from "@executablemd/test-support/failure-journal"; +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; + +const OWNER = "packages/git/tests/issue-markdown.test.ts"; /** The scenarios, in the order a reader meets them. */ const SCENARIOS = [ @@ -59,28 +65,13 @@ function* stated(observation: ScenarioObservation): Operation { + beforeEach(journalBeforeEach(OWNER)); for (const name of SCENARIOS) { const title = `${name} states a contract that holds`; const path = pathOf(name); - if (name === "IssueRead.test.md" || name === "IssueUpsert.test.md") { - it(title, () => - captureTestJournal( - { - owner: "packages/git/tests/issue-markdown.test.ts", - suite: "workflow Issue scenarios", - name: title, - }, - path, - function* (stream) { - yield* stated(yield* runScenario(path, stream)); - }, - ), - ); - } else { - it(title, function* () { - yield* stated(yield* runScenario(path)); - }); - } + it(title, function* () { + yield* stated(yield* runScenario(path, yield* useJournalStream(path))); + }); } it("IssueDurability.test.md states a contract that holds", function* () { @@ -90,7 +81,8 @@ describe("workflow Issue scenarios", () => { // put a run in the state this scenario is about. yield* fixture.stage("IssueDurability.attempt.stage.md"); yield* fixture.stage("IssueDurability.attempt.stage.md"); - yield* stated(yield* fixture.observe(pathOf("IssueDurability.test.md"))); + const path = pathOf("IssueDurability.test.md"); + yield* stated(yield* fixture.observe(path, yield* useJournalStream(path))); }); it("an issue effect retains where its was written", function* () { @@ -123,7 +115,8 @@ describe("workflow Issue scenarios", () => { const fixture = yield* useScenarioFixture(); yield* fixture.stage("IssueReadDurability.attempt.stage.md"); yield* fixture.stage("IssueReadDurability.attempt.stage.md"); - yield* stated(yield* fixture.observe(pathOf("IssueReadDurability.test.md"))); + const path = pathOf("IssueReadDurability.test.md"); + yield* stated(yield* fixture.observe(path, yield* useJournalStream(path))); }); // Each hazard is its own run. They cannot share one: a process that ended @@ -140,7 +133,8 @@ describe("workflow Issue scenarios", () => { const fixture = yield* useScenarioFixture(); yield* fixture.stage(`${hazard}.attempt.stage.md`); yield* fixture.stage(`${hazard}.attempt.stage.md`); - yield* stated(yield* fixture.observe(pathOf(`${hazard}.test.md`))); + const path = pathOf(`${hazard}.test.md`); + yield* stated(yield* fixture.observe(path, yield* useJournalStream(path))); }); } @@ -149,7 +143,8 @@ describe("workflow Issue scenarios", () => { // Staged twice so the second attempt shows there was nothing to restore. yield* fixture.stage("IssueReadSubstituted.attempt.stage.md"); yield* fixture.stage("IssueReadSubstituted.attempt.stage.md"); - yield* stated(yield* fixture.observe(pathOf("IssueReadSubstituted.test.md"))); + const path = pathOf("IssueReadSubstituted.test.md"); + yield* stated(yield* fixture.observe(path, yield* useJournalStream(path))); }); it("IssueRecovery.test.md states a contract that holds", function* () { @@ -157,7 +152,8 @@ describe("workflow Issue scenarios", () => { // The interrupted attempt first, then one that inherits what it left. yield* fixture.stage("IssueRecovery.attempt.stage.md"); yield* fixture.stage("IssueRecovery.attempt.stage.md"); - yield* stated(yield* fixture.observe(pathOf("IssueRecovery.test.md"))); + const path = pathOf("IssueRecovery.test.md"); + yield* stated(yield* fixture.observe(path, yield* useJournalStream(path))); }); }); @@ -177,59 +173,81 @@ describe("workflow Issue scenarios", () => { */ describe("scenario sessions", () => { const sessions = (name: string) => pathOf(`sessions/${name}`); + const owner = "packages/git/tests/issue-markdown.test.ts"; + const suite = "scenario sessions"; - it("shares one tracker between documents that each get their own session", function* () { + const sharedTracker = "shares one tracker between documents that each get their own session"; + it(sharedTracker, function* () { const fixture = yield* useScenarioFixture(); - const first = yield* fixture.observe(sessions("First.test.md")); - const second = yield* fixture.observe(sessions("Second.test.md")); - - // Non-empty and disjoint: each document reports its own test and only - // its own, which is what a session per execution buys. A cumulative - // session would show the first document's result in the second's. - expect(first.results.map((result) => result.name)).toEqual([ - "the first document sees the tracker it filed on", - ]); - expect(second.results.map((result) => result.name)).toEqual([ - "the second document meets the first document's tracker", - ]); - - // Both passed, and the second's passing is the shared half: it asserted - // on an issue the first document filed, so the fixture outlived the - // scope the first session closed. - expect(failures(first)).toEqual([]); - expect(failures(second)).toEqual([]); - expect(first.outcome.ok).toBe(true); - expect(second.outcome.ok).toBe(true); - expect(fixture.server.issues.map((issue) => issue.title)).toEqual([ - "Filed by the first document", - ]); + const testCase = { owner, suite, name: sharedTracker }; + const firstPath = sessions("First.test.md"); + const secondPath = sessions("Second.test.md"); + yield* captureTestJournal(testCase, firstPath, function* (firstStream) { + const first = yield* fixture.observe(firstPath, firstStream); + yield* captureTestJournal(testCase, secondPath, function* (secondStream) { + const second = yield* fixture.observe(secondPath, secondStream); + + // Non-empty and disjoint: each document reports its own test and only + // its own, which is what a session per execution buys. A cumulative + // session would show the first document's result in the second's. + expect(first.results.map((result) => result.name)).toEqual([ + "the first document sees the tracker it filed on", + ]); + expect(second.results.map((result) => result.name)).toEqual([ + "the second document meets the first document's tracker", + ]); + + // Both passed, and the second's passing is the shared half: it asserted + // on an issue the first document filed, so the fixture outlived the + // scope the first session closed. + expect(failures(first)).toEqual([]); + expect(failures(second)).toEqual([]); + expect(first.outcome.ok).toBe(true); + expect(second.outcome.ok).toBe(true); + expect(fixture.server.issues.map((issue) => issue.title)).toEqual([ + "Filed by the first document", + ]); + }); + }); }); - it("fails the document whose assertion fails, and no other", function* () { + const independent = "fails the document whose assertion fails, and no other"; + it(independent, function* () { const fixture = yield* useScenarioFixture(); - const failing = yield* fixture.observe(sessions("Failing.test.md")); - expect(failing.results.map((result) => result.status)).toEqual(["fail"]); - expect(failing.outcome.ok).toBe(false); + const testCase = { owner, suite, name: independent }; + const failingPath = sessions("Failing.test.md"); + yield* captureTestJournal(testCase, failingPath, function* (stream) { + const failing = yield* fixture.observe(failingPath, stream); + expect(failing.results.map((result) => result.status)).toEqual(["fail"]); + expect(failing.outcome.ok).toBe(false); + }); // The next document is unharmed, which is the claim that matters: a // failure belongs to the session that recorded it, so a passing document // observed after a failing one still settles Ok with its own result. - const after = yield* fixture.observe(sessions("Passing.test.md")); - expect(after.results.map((result) => result.status)).toEqual(["pass"]); - expect(after.outcome.ok).toBe(true); + const passingPath = sessions("Passing.test.md"); + yield* captureTestJournal(testCase, passingPath, function* (stream) { + const after = yield* fixture.observe(passingPath, stream); + expect(after.results.map((result) => result.status)).toEqual(["pass"]); + expect(after.outcome.ok).toBe(true); + }); }); - it("fails a document that discovered no tests at all", function* () { + const noTests = "fails a document that discovered no tests at all"; + it(noTests, function* () { const fixture = yield* useScenarioFixture(); // The hazard this arrangement exists to remove: activation that did not // take renders a document full of passing assertion reports and collects // nothing. The session's completion policy is what makes that an `Err` // rather than a silent success. Tracked separately as // taras/executable.md#523 for the testing API itself. - const empty = yield* fixture.observe(sessions("Untested.md")); - expect(empty.results).toEqual([]); - expect(empty.outcome.ok).toBe(false); + const path = sessions("Untested.md"); + yield* captureTestJournal({ owner, suite, name: noTests }, path, function* (stream) { + const empty = yield* fixture.observe(path, stream); + expect(empty.results).toEqual([]); + expect(empty.outcome.ok).toBe(false); + }); }); }); diff --git a/packages/git/tests/pull-request-read.test.ts b/packages/git/tests/pull-request-read.test.ts index 644c8a2ad..764e439c3 100644 --- a/packages/git/tests/pull-request-read.test.ts +++ b/packages/git/tests/pull-request-read.test.ts @@ -10,7 +10,12 @@ * contacts a network host, and no case reads a developer's real credential. */ -import { describe, it } from "@executablemd/test-support/bdd"; +import { beforeEach, describe, it } from "@executablemd/test-support/bdd"; +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; import { expect } from "@executablemd/test-support/expect"; import { call, @@ -68,6 +73,8 @@ import type { GitHubRepositoryName, } from "../src/deno/composition/github.ts"; +const OWNER = "packages/git/tests/pull-request-read.test.ts"; + const ENDPOINT = "https://api.github.test"; /** A body that makes the transport raise rather than answer. */ @@ -505,6 +512,7 @@ function retainedItems(events: DurableEvent[]): Json { } describe("Tier PRR — pull-request evidence", () => { + beforeEach(journalBeforeEach("packages/git/tests/pull-request-read.test.ts")); it("PRR3: every body and every check survives normalization byte for byte", function* () { const body = " Two spaces, a\ttab, and a trailing newline.\n"; const host = server({ @@ -1049,7 +1057,7 @@ describe("Tier PRR — pull-request evidence", () => { }); it("PRR13: an ordinary run resolves none of the three", function* () { - const stream = new InMemoryStream(); + const stream = yield* useJournalStream(""); const failure = yield* raised( collect( yield* execute({ @@ -1669,7 +1677,8 @@ describe("Tier PRR — pull-request evidence", () => { * are measured, because a refusal is published through the same boundary a * success is. */ - it("PRR24a: an admitted read is appended only after its session has closed", function* () { + const publicationOrder = "PRR24a: an admitted read is appended only after its session has closed"; + it(publicationOrder, function* () { for (const outcome of ["success", "refusal"] as const) { const host = outcome === "success" @@ -1677,39 +1686,44 @@ describe("Tier PRR — pull-request evidence", () => { : server({ [REVIEWS]: { status: 500, body: "{}" } }); const order: string[] = []; const gate = gatedAccess(host.access, order); - const stream = new InMemoryStream(); - noteAppends(stream, order); + yield* captureTestJournal( + { owner: OWNER, name: publicationOrder }, + "", + function* (stream) { + noteAppends(stream, order); - let returned = false; - const document = `\n\n{reviews.length} reviews\n`; + let returned = false; + const document = `\n\n{reviews.length} reviews\n`; - try { - const task = yield* spawn(function* () { - yield* raised(retainedRead(stream, document, gate.source)); - returned = true; - }); + try { + const task = yield* spawn(function* () { + yield* raised(retainedRead(stream, document, gate.source)); + returned = true; + }); - yield* gate.closing; - // Held: the transport is over, and nothing has been acknowledged. - expect(order).toEqual(["open", "closing"]); - expect(returned).toBe(false); - expect(readYields(stream.snapshot())).toEqual([]); + yield* gate.closing; + // Held: the transport is over, and nothing has been acknowledged. + expect(order).toEqual(["open", "closing"]); + expect(returned).toBe(false); + expect(readYields(stream.snapshot())).toEqual([]); - gate.release(); - yield* task; - } finally { - // Released whatever happened, so a failed assertion fails rather than - // leaving a suite waiting on a gate nobody will open. - gate.release(); - } + gate.release(); + yield* task; + } finally { + // Released whatever happened, so a failed assertion fails rather than + // leaving a suite waiting on a gate nobody will open. + gate.release(); + } - expect(order).toEqual(["open", "closing", "closed", "appended"]); - expect(returned).toBe(true); + expect(order).toEqual(["open", "closing", "closed", "appended"]); + expect(returned).toBe(true); - // One read either way, and the outcome the collection actually had. - const retained = readYields(stream.snapshot()); - expect(retained).toHaveLength(1); - expect(retained[0]?.result?.status).toBe(outcome === "success" ? "ok" : "err"); + // One read either way, and the outcome the collection actually had. + const retained = readYields(stream.snapshot()); + expect(retained).toHaveLength(1); + expect(retained[0]?.result?.status).toBe(outcome === "success" ? "ok" : "err"); + }, + ); } }); @@ -1783,51 +1797,53 @@ describe("Tier PRR — pull-request evidence", () => { * rather than merely started. The gate supplies both: the halt does not * settle while cleanup is held, and no read reaches the journal at all. */ - it("PRR26a: a cancelled read joins its held cleanup and publishes nothing", function* () { - const entered = withResolvers(); - const order: string[] = []; - const access: GitHubAccess = { - endpoint: ENDPOINT, - // deno-lint-ignore require-yield - *token(): Operation { - return "t"; - }, - *send(): Operation { - entered.resolve(); - yield* suspend(); - throw new Error("unreachable"); - }, - }; - const gate = gatedAccess(access, order); - const stream = new InMemoryStream(); - noteAppends(stream, order); - const document = `\n`; + const cancelledRead = "PRR26a: a cancelled read joins its held cleanup and publishes nothing"; + it(cancelledRead, function* () { + yield* captureTestJournal({ owner: OWNER, name: cancelledRead }, "", function* (stream) { + const entered = withResolvers(); + const order: string[] = []; + const access: GitHubAccess = { + endpoint: ENDPOINT, + // deno-lint-ignore require-yield + *token(): Operation { + return "t"; + }, + *send(): Operation { + entered.resolve(); + yield* suspend(); + throw new Error("unreachable"); + }, + }; + const gate = gatedAccess(access, order); + noteAppends(stream, order); + const document = `\n`; - let halted = false; - try { - const task = yield* spawn(() => raised(retainedRead(stream, document, gate.source))); - yield* entered.operation; + let halted = false; + try { + const task = yield* spawn(() => raised(retainedRead(stream, document, gate.source))); + yield* entered.operation; - const halting = yield* spawn(function* () { - yield* task.halt(); - halted = true; - }); + const halting = yield* spawn(function* () { + yield* task.halt(); + halted = true; + }); - yield* gate.closing; - // The halt has reached this session's teardown and is waiting on it. - expect(halted).toBe(false); - expect(order).toEqual(["open", "closing"]); + yield* gate.closing; + // The halt has reached this session's teardown and is waiting on it. + expect(halted).toBe(false); + expect(order).toEqual(["open", "closing"]); - gate.release(); - yield* halting; - } finally { - gate.release(); - } + gate.release(); + yield* halting; + } finally { + gate.release(); + } - expect(halted).toBe(true); - expect(order).toEqual(["open", "closing", "closed"]); - // Cancellation stays cancellation: no success, and no failed read either. - expect(readYields(stream.snapshot())).toEqual([]); + expect(halted).toBe(true); + expect(order).toEqual(["open", "closing", "closed"]); + // Cancellation stays cancellation: no success, and no failed read either. + expect(readYields(stream.snapshot())).toEqual([]); + }); }); it("PRR32: a received envelope carrying one member more retains the failure", function* () { diff --git a/packages/test-agent/tests/components.test.ts b/packages/test-agent/tests/components.test.ts index 6ce14f428..ced4bf183 100644 --- a/packages/test-agent/tests/components.test.ts +++ b/packages/test-agent/tests/components.test.ts @@ -5,6 +5,7 @@ * clean suspension at teardown — driven through the component surface * with real workers. */ +import { captureTestJournal } from "@executablemd/test-support/failure-journal"; import { describe, it, beforeAll } from "@executablemd/test-support/bdd"; import { useTempFileCompiler } from "@executablemd/core"; import { expect } from "@executablemd/test-support/expect"; @@ -22,7 +23,6 @@ import type { AnswerConfiguration } from "@executablemd/core/host"; import { executeInstalled } from "@executablemd/core/host"; import { API } from "@executablemd/runtime"; import { InMemoryStream } from "@executablemd/durable-streams"; -import { captureTestJournal } from "@executablemd/test-support/failure-journal"; import { installTestingComponents, testHarnessInstallation, @@ -58,6 +58,16 @@ interface RunOptions { const OWNER = "packages/test-agent/tests/components.test.ts"; +function captureDocument( + name: string, + body: (stream: InMemoryStream, directory: string) => Operation, +): Operation { + const directory = path.join(os.tmpdir(), `xmd-tv-${randomUUID()}`); + return captureTestJournal({ owner: OWNER, name }, path.join(directory, "doc.md"), (stream) => + body(stream, directory), + ); +} + function* captureFileCase( name: string, directory: string, @@ -160,7 +170,9 @@ const TWO_STAGES = describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResources: false }, () => { beforeAll(() => useTempFileCompiler()); - it("rejects malformed catalogs equivalently in wrapper and nested declarations before provisioning", function* () { + const malformedCatalogs = + "rejects malformed catalogs equivalently in wrapper and nested declarations before provisioning"; + it(malformedCatalogs, function* () { const invalid = [ { selected: "A", options: [] }, { selected: "A", options: [{ id: "A" }, { id: "A" }] }, @@ -175,20 +187,32 @@ describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResou ]; for (const models of invalid) { const declaration = ``; - const wrapper = yield* runDoc({ - "hi.md": HI, - "doc.md": `\n${declaration}\nnever reached\n\n`, + yield* captureDocument(malformedCatalogs, function* (stream, directory) { + const wrapper = yield* runDoc( + { + "hi.md": HI, + "doc.md": `\n${declaration}\nnever reached\n\n`, + }, + undefined, + stream, + directory, + ); + expect(wrapper.result.ok).toBe(false); + }); + + yield* captureDocument(malformedCatalogs, function* (stream, directory) { + const nested = yield* runDoc( + { + "hi.md": HI, + "doc.md": `\n\n\n${declaration}\n\n\n\n`, + }, + { child: true }, + stream, + directory, + ); + expect(nested.results.map((result) => result.status)).toEqual(["fail"]); + expect(nested.output).not.toContain("invalid catalog reached child provisioning"); }); - expect(wrapper.result.ok).toBe(false); - const nested = yield* runDoc( - { - "hi.md": HI, - "doc.md": `\n\n\n${declaration}\n\n\n\n`, - }, - { child: true }, - ); - expect(nested.results.map((result) => result.status)).toEqual(["fail"]); - expect(nested.output).not.toContain("invalid catalog reached child provisioning"); } }); @@ -337,115 +361,161 @@ describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResou ); }); - it("TV4: non-whitespace pre-matcher output and eval imports are configuration failures", function* () { - const preMatcher = yield* runDoc({ - "agents/bad.md": 'leading text\n\n\nok\n', - "doc.md": [ - "", - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + const configurationFailures = + "TV4: non-whitespace pre-matcher output and eval imports are configuration failures"; + it(configurationFailures, function* () { + yield* captureDocument(configurationFailures, function* (stream, directory) { + const preMatcher = yield* runDoc( + { + "agents/bad.md": 'leading text\n\n\nok\n', + "doc.md": [ + "", + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(preMatcher.results.map((entry) => entry.status)).toEqual(["fail"]); }); - expect(preMatcher.results.map((entry) => entry.status)).toEqual(["fail"]); - const dynamicImport = yield* runDoc({ - "agents/dyn.md": [ - '', - "", - "```js eval", - 'const mod = yield* until(import("node:fs"));', - "return String(typeof mod);", - "```", - "", - ].join("\n"), - "doc.md": [ - "", - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + yield* captureDocument(configurationFailures, function* (stream, directory) { + const dynamicImport = yield* runDoc( + { + "agents/dyn.md": [ + '', + "", + "```js eval", + 'const mod = yield* until(import("node:fs"));', + "return String(typeof mod);", + "```", + "", + ].join("\n"), + "doc.md": [ + "", + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(dynamicImport.results.map((entry) => entry.status)).toEqual(["fail"]); }); - expect(dynamicImport.results.map((entry) => entry.status)).toEqual(["fail"]); - const staticImport = yield* runDoc({ - "agents/static.md": [ - '', - "", - "```js eval", - 'import { readFileSync } from "node:fs";', - "return String(typeof readFileSync);", - "```", - "", - ].join("\n"), - "doc.md": [ - "", - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + yield* captureDocument(configurationFailures, function* (stream, directory) { + const staticImport = yield* runDoc( + { + "agents/static.md": [ + '', + "", + "```js eval", + 'import { readFileSync } from "node:fs";', + "return String(typeof readFileSync);", + "```", + "", + ].join("\n"), + "doc.md": [ + "", + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(staticImport.results.map((entry) => entry.status)).toEqual(["fail"]); }); - expect(staticImport.results.map((entry) => entry.status)).toEqual(["fail"]); - - const whitespaceOnly = yield* runDoc({ - "agents/ws.md": "\n \n\t\n" + HI, - "doc.md": [ - "", - ' ', - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + + yield* captureDocument(configurationFailures, function* (stream, directory) { + const whitespaceOnly = yield* runDoc( + { + "agents/ws.md": "\n \n\t\n" + HI, + "doc.md": [ + "", + ' ', + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(whitespaceOnly.results.map((entry) => entry.status)).toEqual(["pass"]); }); - expect(whitespaceOnly.results.map((entry) => entry.status)).toEqual(["pass"]); }); - it("TV5: Markdown wins over TypeScript; missing .ts falls back; existing .ts is unsupported", function* () { - const precedence = yield* runDoc({ - "agents/components/Greeting.md": "markdown wins\n", - "agents/components/Greeting.ts": 'export default function* () { return "ts"; }\n', - "agents/components/Nested/index.md": "index fallback\n", - "agents/both.md": '\n\n\n\n', - "doc.md": [ - "", - ' ', - ' ', - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + const fileResolution = + "TV5: Markdown wins over TypeScript; missing .ts falls back; existing .ts is unsupported"; + it(fileResolution, function* () { + yield* captureDocument(fileResolution, function* (stream, directory) { + const precedence = yield* runDoc( + { + "agents/components/Greeting.md": "markdown wins\n", + "agents/components/Greeting.ts": 'export default function* () { return "ts"; }\n', + "agents/components/Nested/index.md": "index fallback\n", + "agents/both.md": '\n\n\n\n', + "doc.md": [ + "", + ' ', + ' ', + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(precedence.results.map((entry) => entry.status)).toEqual(["pass"]); }); - expect(precedence.results.map((entry) => entry.status)).toEqual(["pass"]); - - const typescript = yield* runDoc({ - "agents/components/Helper.ts": 'export default function* () { return "nope"; }\n', - "agents/ts-behavior.md": '\n\n\n', - "doc.md": [ - "", - ' ', - ' ', - ' ', - " ", - "", - "", - ].join("\n"), + + yield* captureDocument(fileResolution, function* (stream, directory) { + const typescript = yield* runDoc( + { + "agents/components/Helper.ts": 'export default function* () { return "nope"; }\n', + "agents/ts-behavior.md": '\n\n\n', + "doc.md": [ + "", + ' ', + ' ', + ' ', + " ", + "", + "", + ].join("\n"), + }, + undefined, + stream, + directory, + ); + expect(typescript.results.map((entry) => entry.status)).toEqual(["fail"]); }); - expect(typescript.results.map((entry) => entry.status)).toEqual(["fail"]); }); it("TV6: default and named mappings resolve; the agent prop registers additional agents", function* () { @@ -763,7 +833,8 @@ describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResou // A registration is a default, so a repository component of either name wins. // Each fixture declares a props schema: a markdown component without one // accepts no props and is rejected before it can answer. - it("TV15: a repository component overrides either registered name", function* () { + const repositoryOverride = "TV15: a repository component overrides either registered name"; + it(repositoryOverride, function* () { const withProps = (body: string) => [ "---", @@ -779,31 +850,39 @@ describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResou "", ].join("\n"); - const agentRun = yield* runDoc( - { - "doc.md": "\nbody\n\n", - "components/TestAgent.md": withProps("LOCAL TEST AGENT"), - }, - { includes: ["components", "."] }, - ); - expect(agentRun.output).toContain("LOCAL TEST AGENT"); + yield* captureDocument(repositoryOverride, function* (stream, directory) { + const agentRun = yield* runDoc( + { + "doc.md": "\nbody\n\n", + "components/TestAgent.md": withProps("LOCAL TEST AGENT"), + }, + { includes: ["components", "."] }, + stream, + directory, + ); + expect(agentRun.output).toContain("LOCAL TEST AGENT"); + }); // The dotted name addresses a subdirectory, so the override lives at // components/TestAgent/Scenario.md. - const scenarioRun = yield* runDoc( - { - "agents/hi.md": HI, - "doc.md": [ - "", - '', - "", - "", - ].join("\n"), - "components/TestAgent/Scenario.md": withProps("LOCAL SCENARIO"), - }, - { includes: ["components", "."] }, - ); - expect(scenarioRun.output).toContain("LOCAL SCENARIO"); + yield* captureDocument(repositoryOverride, function* (stream, directory) { + const scenarioRun = yield* runDoc( + { + "agents/hi.md": HI, + "doc.md": [ + "", + '', + "", + "", + ].join("\n"), + "components/TestAgent/Scenario.md": withProps("LOCAL SCENARIO"), + }, + { includes: ["components", "."] }, + stream, + directory, + ); + expect(scenarioRun.output).toContain("LOCAL SCENARIO"); + }); }); }); diff --git a/packages/testing/tests/execution-harness.test.ts b/packages/testing/tests/execution-harness.test.ts index 47b6d4dd1..931675ab0 100644 --- a/packages/testing/tests/execution-harness.test.ts +++ b/packages/testing/tests/execution-harness.test.ts @@ -8,7 +8,7 @@ * about production assembly, which is the CLI's contract and is held there. */ -import { describe, it } from "@executablemd/test-support/bdd"; +import { beforeEach, describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { all, createContext, scoped, withResolvers } from "effection"; import type { Operation, Result } from "effection"; @@ -20,7 +20,11 @@ import { useTempDirectory } from "@executablemd/test-support/temp"; import { join } from "node:path"; import process from "node:process"; import { Writable } from "node:stream"; -import { captureTestJournal } from "@executablemd/test-support/failure-journal"; +import { + captureTestJournal, + journalBeforeEach, + useJournalStream, +} from "@executablemd/test-support/failure-journal"; import type { DeliverySink } from "../../cli/src/stdout-delivery.ts"; import { readRecords } from "../../cli/src/repl/journal-records.ts"; import { useStubFs } from "@executablemd/runtime/test"; @@ -87,7 +91,7 @@ function* runHarness( // The harness is a delivery the host attaches, so these tests are // the host: without this, `` is recognized and refused. const execution = yield* executeInstalled( - { path: "README.md", stream: options.stream ?? new InMemoryStream() }, + { path: "README.md", stream: options.stream ?? (yield* useJournalStream("README.md")) }, [testHarnessInstallation(stub.provider)], ); const chunks: string[] = []; @@ -163,6 +167,7 @@ const CHILD_WITH_PROPS = doc( ); describe(RUN_SUITE, () => { + beforeEach(journalBeforeEach(OWNER)); const referenced = "runs a referenced document as a root, with root props and text return"; it(referenced, function* () { yield* captureHarness( @@ -388,6 +393,7 @@ describe(RUN_SUITE, () => { }); describe("child failure", () => { + beforeEach(journalBeforeEach(OWNER)); const FAILING = { "README.md": "", "boom.md": doc("```sh exec", "exit 3", "```"), @@ -439,6 +445,7 @@ describe("child failure", () => { }); describe("display and collection", () => { + beforeEach(journalBeforeEach(OWNER)); it("displays each chunk before the child has settled", function* () { const displayed = withResolvers(); const run = yield* runHarness( @@ -523,6 +530,7 @@ describe("display and collection", () => { }); describe("journal policy", () => { + beforeEach(journalBeforeEach(OWNER)); const CHILD = { "child.md": doc("child body") }; it("allocates no journal for a transient run", function* () { @@ -596,6 +604,7 @@ describe("journal policy", () => { }); describe("what may run a child", () => { + beforeEach(journalBeforeEach(OWNER)); it("refuses outside a canonical ", function* () { const run = yield* runHarness({ "README.md": doc("", '', ""), @@ -753,6 +762,7 @@ describe("what may run a child", () => { }); describe("the harness path", () => { + beforeEach(journalBeforeEach(OWNER)); const DOC = { "README.md": doc( '', @@ -778,7 +788,7 @@ describe("the harness path", () => { const tests = yield* useTesting(); const stub = stubExecutionHost({ files }); const execution = yield* executeInstalled( - { path: "README.md", stream: new InMemoryStream() }, + { path: "README.md", stream: yield* useJournalStream("README.md") }, installations(stub.provider), ); yield* forEach(function* () {}, execution.output); @@ -805,7 +815,7 @@ describe("the harness path", () => { yield* useStubFs(DOC); const tests = yield* useTesting(); const execution = yield* executeInstalled( - { path: "README.md", stream: new InMemoryStream() }, + { path: "README.md", stream: yield* useJournalStream("README.md") }, [testHarnessInstallation()], ); yield* forEach(function* () {}, execution.output); @@ -1021,6 +1031,7 @@ describe("the harness path", () => { * `execution-host-stub.ts` stands in for the CLI. */ describe("child configuration declarations", () => { + beforeEach(journalBeforeEach(OWNER)); const AGENT_PROPS: PropsSchema = { type: "object", properties: { agent: { type: "string" } }, @@ -1150,7 +1161,7 @@ describe("child configuration declarations", () => { const tests = yield* useTesting(); const stub = stubExecutionHost({ files }); const execution = yield* executeInstalled( - { path: "README.md", stream: new InMemoryStream() }, + { path: "README.md", stream: yield* useJournalStream("README.md") }, [testHarnessInstallation(stub.provider, options.declarations ?? [stubDeclaration()])], ); const output = yield* forEach(function* () {}, execution.output); @@ -1536,7 +1547,7 @@ describe("child configuration declarations", () => { onTeardown: () => order.push("the child's teardown"), }); const execution = yield* executeInstalled( - { path: "README.md", stream: new InMemoryStream() }, + { path: "README.md", stream: yield* useJournalStream("README.md") }, [testHarnessInstallation(stub.provider, [stubDeclaration()])], ); const output = yield* forEach(function* () {}, execution.output);