diff --git a/packages/cli/src/test-diagnostics.ts b/packages/cli/src/test-diagnostics.ts
index 2f9858990..d02b71af3 100644
--- a/packages/cli/src/test-diagnostics.ts
+++ b/packages/cli/src/test-diagnostics.ts
@@ -5,11 +5,11 @@
* Two diagnostics, both owned by the command and neither part of the report a
* document renders:
*
- * - **Progress** is a stream of plain lines on stderr, written as work starts
- * and settles: each test, each executable element and each command block,
- * named and located, then the test's final outcome once its teardown has
- * joined. It names things and places; it never carries a captured value, a
- * prop, command text or anything a document rendered.
+ * - **Progress** reports each test's start and recorded outcome after teardown,
+ * accepted Execution invocations as they start and settle, and interrupted
+ * operations on timeout or cancellation. Ordinary elements and blocks are
+ * observed without their own lines. It names things and places; it never
+ * carries a captured value, a prop, command text or rendered content.
* - **Failure journals** are the in-memory journals of a failed root document
* and of each failed nested `` child, saved as JSONL once that
* execution's owner has unwound. A passing run creates no file.
@@ -32,7 +32,7 @@ import type { DurableEvent } from "@executablemd/durable-streams";
import { Test, testing } from "@executablemd/testing";
import type { FailureJournalHost, TestResult } from "@executablemd/testing";
import { randomUUID } from "node:crypto";
-import { isAbsolute, join, resolve } from "node:path";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { deliverWhole } from "./stdout-delivery.ts";
import type { DeliverySink } from "./stdout-delivery.ts";
@@ -79,11 +79,11 @@ export interface ExecutionDiagnostics {
* saved. A fresh directory per invocation keeps concurrent commands sharing
* one artifact root from writing over each other.
*/
-function* artifactDirectory(): Operation {
+function* artifactDirectory(working: string): Operation {
const configured = yield* env(TEST_ARTIFACTS_VARIABLE);
const base =
configured !== undefined && configured.length > 0 ? configured : DEFAULT_TEST_ARTIFACTS;
- const root = isAbsolute(base) ? base : resolve(yield* cwd(), base);
+ const root = isAbsolute(base) ? base : resolve(working, base);
const stamp = new Date().toISOString().replaceAll(":", "-");
return join(root, `${stamp}-${randomUUID()}`);
}
@@ -97,7 +97,9 @@ function* artifactDirectory(): Operation {
*/
export function useTestDiagnostics(sink: DeliverySink): Operation {
return resource(function* (provide) {
- const directory = yield* artifactDirectory();
+ const working = yield* cwd();
+ const directory = yield* artifactDirectory(working);
+ const shown = relativeTo(working);
const owner = yield* useScope();
const lines = createQueue();
let delivery: Result = Ok(undefined);
@@ -128,19 +130,20 @@ export function useTestDiagnostics(sink: DeliverySink): Operation observe(owner, (line) => emit(`${context}${line}`)),
+ observe: () => observe(owner, shown, (line) => emit(`${context}${line}`)),
journal: (failed: (value: T) => boolean): FailureJournalHost => ({
failed,
*save(events: readonly DurableEvent[]) {
yield* ensureDir(directory);
yield* writeTextFile(file, events.map(serializeDurableEvent).join(""));
- emit(`journal saved: ${file} (${label} ${source})`);
+ emit(`journal saved: ${file} (${named})`);
},
// deno-lint-ignore require-yield
*reportSaveFailure(error: Error) {
- emit(`journal not saved for ${label} ${source}: ${error.message}`);
+ emit(`journal not saved for ${named}: ${error.message}`);
},
}),
};
@@ -150,13 +153,29 @@ export function useTestDiagnostics(sink: DeliverySink): Operation string {
+ return (path) => {
+ if (!isAbsolute(path)) {
+ return path;
+ }
+ const inside = relative(working, path);
+ const outside =
+ inside === "" || inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside);
+ return outside ? path : inside;
+ };
+}
+
/** `path:line:column`, `line:column`, or `unknown`, as test results write it. */
-function located(position: Readonly | undefined): string {
+function located(
+ position: Readonly | undefined,
+ shown: (path: string) => string = (path) => path,
+): string {
if (position === undefined) {
return "unknown";
}
const at = `${position.line}:${position.column}`;
- return position.path ? `${position.path}:${at}` : at;
+ return position.path ? `${shown(position.path)}:${at}` : at;
}
/** A command block, by its language and modifier words — never its body or parameters. */
@@ -172,6 +191,8 @@ function blockName(
/** One test being observed, from its start to its recorded outcome. */
interface ObservedTest {
name: string | undefined;
+ shown: string;
+ recorded?: boolean;
interrupted?: string;
}
@@ -182,7 +203,11 @@ interface ObservedTest {
* element's terminal phase is published after the handler that saw it begin
* has already unwound. Every request is delegated once, as it arrived.
*/
-function* observe(owner: Scope, report: (line: string) => void): Operation {
+function* observe(
+ owner: Scope,
+ shown: (path: string) => string,
+ report: (line: string) => void,
+): Operation {
const tests = new Map();
let current: ObservedTest | undefined;
@@ -192,20 +217,24 @@ function* observe(owner: Scope, report: (line: string) => void): Operation
}
};
- const watch = (what: string, request: ComponentExpansionRequest): void => {
+ const watch = (what: string, request: ComponentExpansionRequest, milestone: boolean): void => {
const test = current;
owner.run(function* () {
- let started = false;
+ let active = false;
for (const phase of yield* each(request.phases)) {
- if (!started) {
- started = true;
+ if (milestone && phase.phase === "active") {
+ active = true;
report(`start ${what}`);
}
if (phase.phase === "complete") {
- report(`${phase.result.ok ? "done" : "failed"} ${what}`);
+ if (active) {
+ report(`${phase.result.ok ? "done" : "failed"} ${what}`);
+ }
} else if (phase.phase === "cancelled") {
interrupted(test, what);
- report(`cancelled ${what}`);
+ if (active) {
+ report(`cancelled ${what}`);
+ }
}
yield* each.next();
}
@@ -214,21 +243,17 @@ function* observe(owner: Scope, report: (line: string) => void): Operation
function* block(what: string, work: () => Operation): Operation {
const test = current;
- report(`start ${what}`);
let settled = false;
try {
const value = yield* work();
settled = true;
- report(`done ${what}`);
return value;
} catch (error) {
settled = true;
- report(`failed ${what}`);
throw error;
} finally {
if (!settled) {
interrupted(test, what);
- report(`cancelled ${what}`);
}
}
}
@@ -239,19 +264,23 @@ function* observe(owner: Scope, report: (line: string) => void): Operation
// its name too; identical names at that site record in discovery order.
const index = observed?.findIndex((test) => test.name === result.name) ?? -1;
const test = index < 0 ? undefined : observed?.splice(index, 1)[0];
+ if (test !== undefined) {
+ test.recorded = true;
+ }
if (observed?.length === 0) {
tests.delete(result.location);
}
const name = result.name === undefined ? "test" : `test ${JSON.stringify(result.name)}`;
+ const where = test?.shown ?? shown(result.location);
if (result.status === "pass") {
- report(`pass ${name} at ${result.location}`);
+ report(`pass ${name} at ${where}`);
return;
}
const why = [
result.error?.kind ?? "error",
...(test?.interrupted === undefined ? [] : [`interrupted ${test.interrupted}`]),
];
- report(`fail ${name} at ${result.location} (${why.join("; ")})`);
+ report(`fail ${name} at ${where} (${why.join("; ")})`);
};
yield* Component.around({
@@ -259,16 +288,20 @@ function* observe(owner: Scope, report: (line: string) => void): Operation
// `` is reported by its behavior and its recorded result, which
// know whether it is a test at all and how it finally ended.
if (request.expansion.name !== "Test") {
- watch(`${request.expansion.name} at ${located(request.expansion.position)}`, request);
+ watch(
+ `${request.expansion.name} at ${located(request.expansion.position, shown)}`,
+ request,
+ request.expansion.name === "Execution",
+ );
}
yield* next(request);
},
*applyModifiers([modifiers, context], next) {
- const what = `${blockName(modifiers, context, false)} at ${located(context.position)}`;
+ const what = `${blockName(modifiers, context, false)} at ${located(context.position, shown)}`;
return yield* block(what, () => next(modifiers, context));
},
*applyBoundModifiers([modifiers, context], next) {
- const what = `${blockName(modifiers, context, true)} at ${located(context.position)}`;
+ const what = `${blockName(modifiers, context, true)} at ${located(context.position, shown)}`;
return yield* block(what, () => next(modifiers, context));
},
});
@@ -277,19 +310,32 @@ function* observe(owner: Scope, report: (line: string) => void): Operation
// The same two answers the behavior reads before it runs anything: a
// test that is skipped or nested inside another is not a test start.
if ((yield* testing) && !(yield* Test.operations.inTest)) {
- const location = located((yield* getExpansion()).position);
+ const position = (yield* getExpansion()).position;
+ const location = located(position);
const name = typeof props.name === "string" ? props.name : undefined;
- const test: ObservedTest = { name };
+ const test: ObservedTest = { name, shown: located(position, shown) };
const observed = tests.get(location) ?? [];
observed.push(test);
tests.set(location, observed);
const previous = current;
current = test;
- report(`start test${name === undefined ? "" : ` ${JSON.stringify(name)}`} at ${location}`);
+ const named = name === undefined ? "test" : `test ${JSON.stringify(name)}`;
+ report(`start ${named} at ${test.shown}`);
+ let settled = false;
try {
- return yield* next(props);
+ const value = yield* next(props);
+ settled = true;
+ return value;
+ } catch (error) {
+ settled = true;
+ throw error;
} finally {
current = previous;
+ if (!settled && !test.recorded) {
+ const detail =
+ test.interrupted === undefined ? "" : ` (interrupted ${test.interrupted})`;
+ report(`cancelled ${named} at ${test.shown}${detail}`);
+ }
}
}
return yield* next(props);
diff --git a/packages/cli/tests/test-diagnostics.test.ts b/packages/cli/tests/test-diagnostics.test.ts
index f383d1ea2..e8675715a 100644
--- a/packages/cli/tests/test-diagnostics.test.ts
+++ b/packages/cli/tests/test-diagnostics.test.ts
@@ -159,11 +159,22 @@ function* launch(directory: string, args: string[], artifacts: string): Operatio
function gate(file: string, ...modifiers: string[]): string[] {
return [
["```sh", "exec", ...modifiers].join(" "),
- `while [ ! -f ${file} ]; do sleep 0.05; done`,
+ `: > ${file}.waiting; while [ ! -f ${file} ]; do sleep 0.05; done`,
"```",
];
}
+function* arrived(directory: string, file: string): Operation {
+ const waited = yield* timebox(RUN_TIMEOUT, function* () {
+ while (!(yield* exists(join(directory, file)))) {
+ yield* sleep(20);
+ }
+ });
+ if (waited.timeout) {
+ throw new Error(`${file} was never created in ${directory}`);
+ }
+}
+
const FAILING_CHILD = '\n';
describe("xmd test failure journals", () => {
@@ -309,7 +320,7 @@ describe("xmd test live progress", () => {
});
const artifacts = join(directory, "artifacts");
const run = yield* launch(directory, ["suite.test.md"], artifacts);
- yield* run.waitFor("start sh exec at suite.test.md:4:1");
+ yield* arrived(directory, "never-created.waiting");
const status = yield* run.join();
expect(status.code).toBe(1);
const lines = progress(run.stderr());
@@ -318,7 +329,8 @@ describe("xmd test live progress", () => {
'fail test "interrupted" at suite.test.md:2:1 (timeout; interrupted sh exec at suite.test.md:4:1)',
'pass test "after" at suite.test.md:2:1',
]);
- expect(lines).toContain("cancelled sh exec at suite.test.md:4:1");
+ expect(lines).not.toContain("cancelled sh exec at suite.test.md:4:1");
+ expect(lines).not.toContain("start sh exec at suite.test.md:4:1");
expect(lines.indexOf('start test "interrupted" at suite.test.md:2:1')).toBeLessThan(
lines.indexOf('pass test "before" at suite.test.md:2:1'),
);
@@ -345,24 +357,20 @@ describe("xmd test live progress", () => {
]);
});
- it("reports a waiting test and its operation before the work is released", function* () {
+ it("reports a waiting test before release without ordinary operation chatter", function* () {
const directory = yield* useProject({
"suite.test.md": doc('', "", ...gate("release"), "", ""),
});
const artifacts = join(directory, "artifacts");
const run = yield* launch(directory, ["suite.test.md"], artifacts);
- yield* run.waitFor("start sh exec at suite.test.md:3:1");
- expect(progress(run.stderr())).toEqual([
- 'start test "held" at suite.test.md:1:1',
- "start sh exec at suite.test.md:3:1",
- ]);
+ yield* arrived(directory, "release.waiting");
+ yield* run.waitFor('start test "held" at suite.test.md:1:1');
+ expect(progress(run.stderr())).toEqual(['start test "held" at suite.test.md:1:1']);
yield* writeTextFile(join(directory, "release"), "");
const status = yield* run.join();
expect(status.code).toBe(0);
expect(progress(run.stderr())).toEqual([
'start test "held" at suite.test.md:1:1',
- "start sh exec at suite.test.md:3:1",
- "done sh exec at suite.test.md:3:1",
'pass test "held" at suite.test.md:1:1',
]);
expect(yield* exists(artifacts)).toBe(false);
@@ -402,11 +410,17 @@ describe("xmd test live progress", () => {
'fail test "times out" at suite.test.md:9:1 (timeout; interrupted Execution at suite.test.md:10:1)',
'fail test "bound wait" at suite.test.md:13:1 (timeout; interrupted sh exec as at suite.test.md:15:1)',
]);
- expect(lines).toContain("child 1 ./waits.md: start sh exec at ./waits.md:1:1");
- expect(lines).toContain("child 1 ./waits.md: cancelled sh exec at ./waits.md:1:1");
+ expect(lines).toContain("start Execution at suite.test.md:10:1");
expect(lines).toContain("cancelled Execution at suite.test.md:10:1");
- expect(lines).toContain("start sh exec as at suite.test.md:15:1");
- expect(lines).toContain("cancelled sh exec as at suite.test.md:15:1");
+ expect(
+ lines.some((line) =>
+ /^journal saved: .*child-1\.jsonl \(child 1 \.\/waits\.md\)$/.test(line),
+ ),
+ ).toBe(true);
+ expect(
+ lines.filter((line) => /^(start|done|failed|cancelled) (?!test\b|Execution\b)/.test(line)),
+ ).toEqual([]);
+ expect(lines.some((line) => line.startsWith("child "))).toBe(false);
// Everything a test started is still there after the run failed.
for (const name of ["asserts", "raises", "times out", "bound wait"]) {
expect(lines.filter((line) => line.startsWith(`start test "${name}"`))).toHaveLength(1);
@@ -428,12 +442,7 @@ describe("xmd test live progress", () => {
});
const artifacts = join(directory, "artifacts");
const run = yield* launch(directory, ["suite.test.md", "--include", FIXTURES], artifacts);
- const started = yield* timebox(RUN_TIMEOUT, function* () {
- while (!(yield* exists(join(directory, "teardown-started")))) {
- yield* sleep(20);
- }
- });
- expect(started.timeout).toBe(false);
+ yield* arrived(directory, "teardown-started");
expect(progress(run.stderr()).filter((line) => /^(pass|fail) test/.test(line))).toEqual([]);
yield* writeTextFile(join(directory, "release-teardown"), "");
const status = yield* run.join();
@@ -446,7 +455,7 @@ describe("xmd test live progress", () => {
expect(lines).toContain('start test "later" at suite.test.md:6:1');
});
- it("names elements and nested children without printing captured or rendered content", function* () {
+ it("reports accepted Executions without ordinary chatter or declaration scan failures", function* () {
const directory = yield* useProject({
"child.md": doc(
'canary-child-let',
@@ -471,6 +480,13 @@ describe("xmd test live progress", () => {
"",
"",
'',
+ '',
+ "",
+ "```sh exec",
+ "true",
+ "```",
+ "",
+ "",
"",
),
});
@@ -494,13 +510,15 @@ describe("xmd test live progress", () => {
expect(result.stdout).toContain('expected: "canary-let-content"');
expect(result.stdout).not.toContain("canary-slice-text");
expect(result.stdout).not.toContain("canary-child-let");
- const lines = progress(result.stderr);
- expect(lines).toContain("start sh exec as at suite.test.md:4:1");
- expect(lines).toContain("start Slice at suite.test.md:8:1");
- expect(lines).toContain("start Execution at suite.test.md:10:1");
- expect(lines).toContain("child 1 ./child.md: start sh exec at ./child.md:3:1");
- expect(lines).toContain("child 1 ./child.md: done Slice at ./child.md:7:1");
- expect(lines.at(-1)).toBe('pass test "private" at suite.test.md:1:1');
+ expect(progress(result.stderr)).toEqual([
+ 'start test "private" at suite.test.md:1:1',
+ "start Execution at suite.test.md:10:1",
+ "done Execution at suite.test.md:10:1",
+ "start Execution at suite.test.md:14:1",
+ "done Execution at suite.test.md:14:1",
+ 'pass test "private" at suite.test.md:1:1',
+ ]);
+ expect(yield* exists(artifacts)).toBe(false);
});
it("keeps delivered progress and saves a partial journal when the run is cancelled", function* () {
@@ -509,19 +527,23 @@ describe("xmd test live progress", () => {
});
const artifacts = join(directory, "artifacts");
const run = yield* launch(directory, ["suite.test.md"], artifacts);
- yield* run.waitFor("start sh exec at suite.test.md:3:1");
+ yield* arrived(directory, "never-created.waiting");
+ yield* run.waitFor('start test "cancelled" at suite.test.md:1:1');
process.kill(run.pid, "SIGINT");
yield* run.join();
const lines = progress(run.stderr());
expect(lines.slice(0, 2)).toEqual([
'start test "cancelled" at suite.test.md:1:1',
- "start sh exec at suite.test.md:3:1",
+ 'cancelled test "cancelled" at suite.test.md:1:1 (interrupted sh exec at suite.test.md:3:1)',
]);
- expect(lines).toContain("cancelled sh exec at suite.test.md:3:1");
+ expect(lines.filter((line) => /^(pass|fail) test/.test(line))).toEqual([]);
const saved = yield* journals(artifacts);
expect(fileNames(saved)).toEqual(["root-1.jsonl"]);
const [events] = [...saved.values()];
expect(events!.length).toBeGreaterThan(0);
+ expect(
+ events!.some((event) => event.type === "yield" && event.description.type === "test_result"),
+ ).toBe(false);
expect(lines.at(-1)).toMatch(/^journal saved: .*root-1\.jsonl \(root 1 suite\.test\.md\)$/);
const delivered = run.stderr();
yield* sleep(200);
diff --git a/packages/cli/tests/test-target.test.ts b/packages/cli/tests/test-target.test.ts
index b71771dfb..887f85539 100644
--- a/packages/cli/tests/test-target.test.ts
+++ b/packages/cli/tests/test-target.test.ts
@@ -106,8 +106,9 @@ describe("Tier DT — xmd test targets", { sanitizeOps: false, sanitizeResources
const diagnostic = lines.filter((line) => !line.startsWith("[xmd test] ")).join("\n");
expect(diagnostic).not.toContain("broken.test.md:");
expect(stderr.match(/tests failed:/g)).toEqual(["tests failed:"]);
+ const shown = path.relative(path.resolve(), document);
expect(progress).toContain(
- `[xmd test] fail test "mixed suite failing document" at ${document}:3:1 (assertion)`,
+ `[xmd test] fail test "mixed suite failing document" at ${shown}:3:1 (assertion)`,
);
});
diff --git a/specs/testing-spec.md b/specs/testing-spec.md
index 5e546fd7a..a1a4da1d7 100644
--- a/specs/testing-spec.md
+++ b/specs/testing-spec.md
@@ -83,23 +83,35 @@ order, with completed records replaying in place.
### Progress and failure journals
-While `xmd test` runs, it writes a progress report to stderr, one line per
-change, so a pipe or a CI log shows where a run is before it finishes:
+While `xmd test` runs, it writes a short progress report to stderr, so a pipe or
+CI log shows which test is running and which operation a timeout interrupted:
```text
[xmd test] start test "held" at suite.test.md:1:1
-[xmd test] start sh exec at suite.test.md:3:1
-[xmd test] cancelled sh exec at suite.test.md:3:1
[xmd test] fail test "held" at suite.test.md:1:1 (timeout; interrupted sh exec at suite.test.md:3:1)
[xmd test] journal saved: /work/.xmd/test-artifacts/2026-…-6f1c…/root-1.jsonl (root 1 suite.test.md)
```
A test's start is reported when its behavior begins and its outcome once its
result is recorded, which follows the test's teardown — a component that
-expanded successfully is not a passing test. Between them each executable
-element and command block reports its start and how it settled, so a timeout
-names the operation it interrupted. Work inside a nested `` child is
-labelled with that child, for example `child 1 ./child.md:`. The report names
+expanded successfully is not a passing test. An `` invocation reports
+its start only after resolution and validation accept it, then reports how that
+invocation settles. These milestones do not claim that a child launched or
+passed. An invocation stopped before activation reports no milestone. Ordinary
+components and command blocks produce no start or terminal lines, including on
+failure or cancellation; their observations still identify the operation a test
+timeout interrupted. The CLI's terminal failure summary is unchanged.
+
+When command cancellation unwinds a test before a result is recorded, progress
+reports `cancelled test` with its name, location and interrupted operation when
+one exists. This notice is not a pass/fail verdict and creates no test result.
+It follows the delegated test work's unwind.
+
+Work inside a nested `` child is labelled with that child, for example
+`child 1 ./child.md:`. Absolute source paths inside the command's captured working
+directory are displayed relative to it; authored relative paths and absolute
+paths outside that directory retain their spelling. Canonical result and
+journal locations keep their original identities. The report names
tests, elements, command languages and modifier words, locations and states. It
never carries a captured value, a prop, command or prompt text, an assertion's
operands or anything a document rendered, and the rendered report on stdout is