From d8f4a56fc746f7512748766b974e527ee94063dd Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 06:29:14 -0500 Subject: [PATCH 01/10] feat(agent): retain Prompt usage with canonical Prompt history (#1159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An acknowledged Agent Prompt now keeps the usage its provider reported in its own agent_prompt record, under the existing retention selection. - Core: optional PromptRecord.usage with a closed reader; a subscription-local usage destination in the audited stream, sealed after joined teardown; the optional promptUsage() receiver on the delivered coordinator, preserved by the installation projection; selection under the producing execution's disclosure before the existing append. - ACPX runtime (XMD prompt usage seam): every terminal result carries the generated user-message ID, owning record, prompted ACP session and selected response/quota counters, including timeout-drain recovery and failed finalization. - ACP provider: captures the receiver at subscription, collects live readings from its one reader, reports outside provider-failure handling; version-bound Claude 0.70.0 and Codex 1.6.2 source semantics. - Hosts state their build facts for Deno source, compiled and workflow runs. - examples/agent-usage: the approved calibration driver, scenario documents and README. - specs/acp-client-spec.md §Prompt usage; architecture note. Co-Authored-By: Claude Opus 5.5 --- architecture.md | 2 + examples/agent-usage/README.md | 73 ++ examples/agent-usage/calibrate.ts | 621 +++++++++++++++++ examples/agent-usage/evidence.ts | 459 ++++++++++++ examples/agent-usage/report.ts | 132 ++++ examples/agent-usage/scenarios/cancel.md | 14 + examples/agent-usage/scenarios/failure.md | 13 + .../scenarios/resume-continuation.md | 12 + examples/agent-usage/scenarios/resume-seed.md | 13 + examples/agent-usage/scenarios/subagent.md | 13 + examples/agent-usage/scenarios/success.md | 15 + examples/agent-usage/selection.ts | 108 +++ packages/acp/mod.ts | 1 + packages/acp/src/adapter-snapshots.ts | 20 +- packages/acp/src/events.ts | 16 +- packages/acp/src/provider.ts | 56 +- packages/acp/src/usage.ts | 374 ++++++++++ packages/acp/tests/prompt-usage.test.ts | 297 ++++++++ packages/acp/tests/provider.test.ts | 433 +++++++++++- packages/acp/vendor/acpx/MANIFEST.json | 8 +- packages/acp/vendor/acpx/PROVENANCE.md | 24 + .../generated/queue-owner-runtime-B0mPqOu4.js | 8 +- .../acp/vendor/acpx/generated/runtime.d.ts | 34 +- packages/acp/vendor/acpx/generated/runtime.js | 54 +- packages/cli/src/agent-stack.ts | 1 + packages/cli/src/compiled.ts | 12 +- packages/cli/src/deno-workflow.ts | 13 +- packages/cli/src/deno.ts | 12 +- packages/cli/src/session-coordinator.ts | 4 +- packages/cli/src/workflow-agent.ts | 4 + .../cli/tests/agent-usage-calibration.test.ts | 197 ++++++ packages/cli/tests/prompt-usage-hosts.test.ts | 245 +++++++ packages/cli/tests/support/fake-acp.ts | 11 +- packages/core/mod.ts | 11 + .../core/src/agent/function-components.ts | 16 + packages/core/src/agent/journal.ts | 106 ++- packages/core/src/agent/launch-coordinator.ts | 14 + packages/core/src/agent/provider-api.ts | 6 + packages/core/src/agent/usage.ts | 569 +++++++++++++++ packages/core/tests/agent-usage.test.ts | 656 ++++++++++++++++++ .../tests/workflow-agent-checkpoints.test.ts | 125 +++- scripts/runtime-test-exclusions.ts | 12 + scripts/tests/acpx-vendor.test.ts | 1 + specs/acp-client-spec.md | 133 +++- 44 files changed, 4904 insertions(+), 44 deletions(-) create mode 100644 examples/agent-usage/README.md create mode 100644 examples/agent-usage/calibrate.ts create mode 100644 examples/agent-usage/evidence.ts create mode 100644 examples/agent-usage/report.ts create mode 100644 examples/agent-usage/scenarios/cancel.md create mode 100644 examples/agent-usage/scenarios/failure.md create mode 100644 examples/agent-usage/scenarios/resume-continuation.md create mode 100644 examples/agent-usage/scenarios/resume-seed.md create mode 100644 examples/agent-usage/scenarios/subagent.md create mode 100644 examples/agent-usage/scenarios/success.md create mode 100644 examples/agent-usage/selection.ts create mode 100644 packages/acp/src/usage.ts create mode 100644 packages/acp/tests/prompt-usage.test.ts create mode 100644 packages/cli/tests/agent-usage-calibration.test.ts create mode 100644 packages/cli/tests/prompt-usage-hosts.test.ts create mode 100644 packages/core/src/agent/usage.ts create mode 100644 packages/core/tests/agent-usage.test.ts diff --git a/architecture.md b/architecture.md index f44530996..d7c42aebe 100644 --- a/architecture.md +++ b/architecture.md @@ -134,6 +134,8 @@ External mutations cannot join that transaction. Their owning provider contract Git-host effects, Issue effects, and Agent Prompts have distinct provider contracts. Completed records restore their historical outcomes without contacting those providers. Live attempts acquire authentication for their own invocation. +Agent Prompt usage is retained with the acknowledged Prompt under the producing execution's disclosure policy. Observation neither authorizes execution nor publishes an independent record. The [ACP client specification](specs/acp-client-spec.md#prompt-usage) owns the receiver's subscription lifetime and retained data. + A compatible history fork creates another run from a retained prefix and checkpoint. It preserves inherited evidence and does not rewind external systems. An XMD artifact exports immutable evidence rather than a writable continuation of its source run. Command display and command retention are separate. Retained channels describe bytes received after enclosing host middleware; downstream display suppression does not alter that record. diff --git a/examples/agent-usage/README.md b/examples/agent-usage/README.md new file mode 100644 index 000000000..08454c886 --- /dev/null +++ b/examples/agent-usage/README.md @@ -0,0 +1,73 @@ +# Calibrate Prompt usage for one bundled Agent + +Every acknowledged Agent Prompt keeps the usage its provider reported in its +canonical journal record. This experiment tells you what those numbers cover on +the build you are measuring: whether a reported total is the whole outer Prompt, +only its final internal request, or something the source cannot say. It sends a +few real Prompts through one production XMD host and compares what the journal +retained with ACPX's saved copy and with the native agent's own account. + +Prepare this checkout with `deno task setup`, and install and authenticate the +native agent separately. Nothing here installs an agent or obtains credentials. + +## Run one experiment + +Each invocation selects exactly one host, one bundled Agent and one scenario, +and may spend tokens: + +```sh +deno run --allow-all --frozen examples/agent-usage/calibrate.ts --host=source --agent=claude --scenario=success --output=/private/claude-source-success --allow-paid +deno run --allow-all --frozen examples/agent-usage/calibrate.ts --host=compiled --xmd=/absolute/path/to/xmd --agent=codex --scenario=success --output=/private/codex-compiled-success --allow-paid +``` + +To run another scenario, change only `--scenario`. `--output` must name a +directory that does not exist yet; an experiment never merges with or +overwrites earlier evidence. `--help` prints the grammar and starts nothing. + +| Scenario | Outer Prompts | What it must show | +| --- | --- | --- | +| `success` | 2 | One Session; two separate file reads in the first Prompt, then recall without tools. | +| `resume` | 2 | A seed, a joined shutdown, then a continuation of the exact retained Session in a separate invocation. | +| `failure` | 1 | An attempted native write refused by the host under `--deny-all`. | +| `cancel` | 1 | Cancellation after the Agent said `USAGE_CALIBRATION_READY`, while its work was still active. | +| `subagent` | 1 | One native subagent where supported, or the specific gap. | + +Each Prompt is attempted once. Nothing is retried, repaired or restarted, and no +other scenario runs. Each Prompt has 60 seconds of work, and no invocation is +admitted after 300 seconds; either deadline interrupts the host's own work and +then waits for it to finish. + +The scenario documents in `scenarios/` are what the host runs; read them to see +exactly what is asked. The source host is `packages/cli/src/deno.ts` from this +checkout; the compiled host is the exact `--xmd` executable. Both run with +`--journal`, so the history they write is ordinary canonical XMD history. + +## Read the result + +The command ends with one line: + +```text +Calibration completed. Read /report.md. +Calibration completed with accounting limitations. Read /report.md. +Calibration incomplete. Read /report.md before attempting another run. +``` + +`report.md` and `report.json` state the same result: the builds measured, each +acknowledged Prompt's source-labelled readings and coverage, and one row per +check — usage retained, request-to-message association, ACPX's saved copy, +the independent native comparison, the scenario's own behavior, history +readback and joined teardown. Each row is `verified`, `limited`, `unavailable`, +`failed` or `not_run`, with a fixed reason. + +A `limited` row is a measured bound, such as `final-request-only`: the reported +total matched only the last internal request of a Prompt that made several. +Agreement with ACPX's saved copy proves the transport, not that a total covers +the whole Prompt. A scenario that did not happen — reads combined, no +cancellation marker, a write never attempted — makes the run incomplete rather +than a limitation. + +The shareable report holds counts, version tokens, digests, reviewed model +names and fixed reasons; no identity, path, marker, prompt or reply text. The +private files beside it — the child journals, `support/` and +`evidence-index.json` — are evidence for whoever ran the experiment, not a +second account of usage and not input to another run. diff --git a/examples/agent-usage/calibrate.ts b/examples/agent-usage/calibrate.ts new file mode 100644 index 000000000..e86d301f9 --- /dev/null +++ b/examples/agent-usage/calibrate.ts @@ -0,0 +1,621 @@ +/** + * Measure one bundled Agent through one production XMD host. + * + * Markdown owns what is asked: each scenario document under `scenarios/` + * sequences its own Prompts in one ``. This program owns only what a + * document cannot: starting the selected host as a child, cancelling it at the + * moment a scenario names, joining it, and comparing the canonical history it + * left with the runtime's saved copy and the native agent's own account. + */ + +import { ensure, exit, main, race, scoped, sleep, spawn, until } from "effection"; +import type { Operation, Stream, Task } from "effection"; +import { exec, Stdio } from "@effectionx/process"; +import { ensureDir, exists, readdir, readTextFile, stat, writeTextFile } from "@effectionx/fs"; +import { mkdir, readFile } from "node:fs/promises"; +import { createHash, randomBytes } from "node:crypto"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; +import type { PromptUsage } from "@executablemd/core"; +import { ALLOWANCE, HELP, parseSelection, Refusal } from "./selection.ts"; +import type { Selection } from "./selection.ts"; +import { + acpxAgreement, + check, + claudeNative, + codexNative, + nativeComparison, + readAcknowledged, +} from "./evidence.ts"; +import type { Acknowledged, Check, NativeSession } from "./evidence.ts"; +import { renderMarkdown } from "./report.ts"; +import type { PromptReport, Report } from "./report.ts"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CHECKOUT = resolve(HERE, "..", ".."); +const SCENARIOS = join(HERE, "scenarios"); +/** The per-Prompt work deadline each scenario document's provider applies. */ +const PROMPT_SECONDS = 60; +/** No invocation is admitted after this many seconds from the start. */ +const ADMISSION_SECONDS = 300; +const READY = "USAGE_CALIBRATION_READY"; + +/** One child invocation a scenario plans. */ +interface Invocation { + readonly document: string; + readonly props: Readonly>; + readonly journal: string; + readonly prompts: number; + readonly denyAll?: boolean; + /** Cancel once the live Agent output says this, while the work is still active. */ + readonly cancelOn?: string; +} + +/** How one child ended, observed by joining it. */ +interface Joined { + readonly started: boolean; + readonly code?: number; + readonly signal?: string; + /** The cancellation marker was seen while the work was still active, and cancellation was sent. */ + readonly cancelledAfterReady?: boolean; + readonly deadline?: boolean; +} + +function hex(): string { + return randomBytes(6).toString("hex"); +} + +function* digestOf(path: string): Operation { + return `sha256:${createHash("sha256") + .update(yield* until(readFile(path))) + .digest("hex")}`; +} + +/** The argv of the selected host, ready for an XMD command. */ +function hostCommand(selection: Selection): string[] { + if (selection.host === "compiled" && selection.xmd !== undefined) { + return [selection.xmd]; + } + return [process.execPath, "run", "--allow-all", "--frozen", join(CHECKOUT, "packages/cli/src/deno.ts")]; +} + +/** Run one short non-prompt command and keep its first line, or nothing. */ +function* firstLine(argv: string[], cwd: string): Operation { + try { + const result = yield* scoped(function* () { + const child = yield* exec(argv[0], { arguments: argv.slice(1), cwd }); + let out = ""; + const reading = yield* spawn(function* () { + for (const chunk of yield* eachChunk(child.stdout)) { + out += chunk; + } + }); + const status = yield* child.join(); + yield* reading; + return status.code === 0 ? out : undefined; + }); + return result?.split("\n").find((line) => line.trim().length > 0)?.trim(); + } catch { + return undefined; + } +} + +/** Every chunk of an output stream, decoded, until it closes. */ +function* eachChunk(stream: Stream): Operation { + const chunks: string[] = []; + const subscription = yield* stream; + const decoder = new TextDecoder(); + let next = yield* subscription.next(); + while (!next.done) { + chunks.push(decoder.decode(next.value, { stream: true })); + next = yield* subscription.next(); + } + return chunks; +} + +const VERSION_TOKEN = /\b\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?\b/u; + +/** What the run measured about the builds before any Prompt was sent. */ +interface Measured { + readonly xmdVersion?: string; + readonly xmdDigest?: string; + readonly sourceRevision?: string; + readonly sourceClean?: boolean; + readonly nativeVersion?: string; +} + +function* measure(selection: Selection, cwd: string): Operation { + const version = (yield* firstLine([...hostCommand(selection), "--version"], cwd))?.match(VERSION_TOKEN)?.[0]; + // The executable the bundled adapter would bind, when the host names one. + const bound = + selection.agent === "claude" ? process.env.CLAUDE_CODE_EXECUTABLE : process.env.CODEX_PATH; + const native = (yield* firstLine([bound ?? selection.agent, "--version"], cwd))?.match(VERSION_TOKEN)?.[0]; + if (selection.host === "compiled" && selection.xmd !== undefined) { + return { + ...(version === undefined ? {} : { xmdVersion: version }), + xmdDigest: yield* digestOf(selection.xmd), + ...(native === undefined ? {} : { nativeVersion: native }), + }; + } + const revision = yield* firstLine(["git", "-C", CHECKOUT, "rev-parse", "HEAD"], cwd); + const status = yield* scoped(function* () { + const child = yield* exec("git", { arguments: ["-C", CHECKOUT, "status", "--porcelain"], cwd }); + let out = ""; + const reading = yield* spawn(function* () { + out = (yield* eachChunk(child.stdout)).join(""); + }); + const joined = yield* child.join(); + yield* reading; + return joined.code === 0 ? out : undefined; + }); + return { + ...(version === undefined ? {} : { xmdVersion: version }), + ...(revision === undefined || !/^[0-9a-f]{40}$/u.test(revision) ? {} : { sourceRevision: revision }), + ...(status === undefined ? {} : { sourceClean: status.trim().length === 0 }), + ...(native === undefined ? {} : { nativeVersion: native }), + }; +} + +/** + * Whether ACPX's own live checkpoint of a session shows the readiness marker. + * + * The provider checkpoints a turn's session updates while it runs, so the + * marker in the saved Agent text, in the record whose user message names this + * run's private fixture, is the earliest evidence a host has that the Agent + * actually said it while its work is still active. Nothing is copied. + */ +function* readyInAcpx(acpxRoot: string, fixture: string, since: number): Operation { + const sessions = join(acpxRoot, "sessions"); + if (!(yield* exists(sessions))) { + return false; + } + for (const name of yield* readdir(sessions)) { + if (!name.endsWith(".json")) { + continue; + } + const path = join(sessions, name); + try { + if ((yield* stat(path)).mtimeMs < since) { + continue; + } + const text = yield* readTextFile(path); + if (!text.includes(fixture) || !text.includes(READY)) { + continue; + } + const record: unknown = JSON.parse(text); + const messages = Reflect.get(Object(record), "messages"); + if (!Array.isArray(messages)) { + continue; + } + const named = messages.some((message) => JSON.stringify(Reflect.get(Object(message), "User") ?? null).includes(fixture)); + const said = messages.some((message) => JSON.stringify(Reflect.get(Object(message), "Agent") ?? null).includes(READY)); + if (named && said) { + return true; + } + } catch { + // A record being replaced while it is read is read again next time. + } + } + return false; +} + +/** + * Run one child host invocation to its joined end. + * + * A deadline or readiness marker sends the child an interrupt, which is the + * host's own structured cancellation, and then joins it: nothing is detached + * or killed to make a report timely. + */ +function* invoke( + selection: Selection, + invocation: Invocation, + cwd: string, + deadlineAt: number, + stderrPath: string, +): Operation { + const argv = [ + ...hostCommand(selection), + "run", + invocation.document, + "--journal", + invocation.journal, + ...(invocation.denyAll === true ? ["--deny-all"] : []), + ...Object.entries(invocation.props).flatMap(([name, value]) => [`--props-${name}`, value]), + ]; + return yield* scoped(function* () { + const started = Date.now(); + const child = yield* exec(argv[0], { arguments: argv.slice(1), cwd }); + let stderr = ""; + const drains: Task[] = [ + yield* spawn(function* () { + yield* eachChunk(child.stdout); + }), + yield* spawn(function* () { + stderr = (yield* eachChunk(child.stderr)).join(""); + }), + ]; + yield* ensure(function* () { + yield* writeTextFile(stderrPath, stderr.slice(0, 8000)); + }); + let cancelledAfterReady = false; + let deadline = false; + const workDeadline = Math.min(deadlineAt, started + invocation.prompts * PROMPT_SECONDS * 1000 + 30_000); + const status = yield* race([ + child.join(), + (function* (): Operation { + while (true) { + yield* sleep(250); + if (invocation.cancelOn !== undefined && !cancelledAfterReady) { + if (yield* readyInAcpx(join(homedir(), ".acpx"), invocation.cancelOn, started)) { + cancelledAfterReady = true; + process.kill(child.pid, "SIGINT"); + } + } + if (!deadline && Date.now() > workDeadline) { + deadline = true; + process.kill(child.pid, "SIGINT"); + } + } + })(), + ]); + for (const drain of drains) { + yield* drain; + } + return { + started: true, + ...(status.code === undefined ? {} : { code: status.code }), + ...(status.signal === undefined ? {} : { signal: status.signal }), + ...(invocation.cancelOn === undefined ? {} : { cancelledAfterReady }), + ...(deadline ? { deadline } : {}), + }; + }); +} + +function* nativeFor(agent: Selection["agent"], record: Acknowledged | undefined): Operation { + const id = record?.record.agentSessionId; + if (id === undefined) { + return undefined; + } + try { + return agent === "claude" ? yield* claudeNative(homedir(), id) : yield* codexNative(homedir(), id); + } catch { + return undefined; + } +} + +/** The shareable view of one usage value: counts, fixed semantics and categorical coverage. */ +function shareable(usage: PromptUsage | undefined): PromptReport["usage"] { + if (usage === undefined) { + return undefined; + } + return { + readings: usage.readings.map((reading) => { + const { model, ...rest } = reading; + return model !== undefined && /^[A-Za-z0-9._:\-[\]]{1,80}$/u.test(model) ? reading : rest; + }), + coverage: usage.coverage, + }; +} + +function* runScenario(selection: Selection, output: string): Operation { + const fixture = join(output, "fixture"); + const support = join(output, "support"); + yield* ensureDir(fixture); + yield* ensureDir(support); + const markers = { a: `USAGE-A-${hex()}`, b: `USAGE-B-${hex()}`, r: `USAGE-R-${hex()}` }; + const fileA = join(fixture, "a.txt"); + const fileB = join(fixture, "b.txt"); + const deniedFile = join(fixture, "denied.txt"); + yield* writeTextFile(fileA, `${markers.a}\n`); + yield* writeTextFile(fileB, `${markers.b}\n`); + const session = `usage-calibration-${hex()}`; + const base = { agent: selection.agent, session }; + const journal = (name: string) => join(output, `${name}.jsonl`); + + const plan: Invocation[] = (() => { + switch (selection.scenario) { + case "success": + return [{ document: join(SCENARIOS, "success.md"), props: { ...base, first: fileA, second: fileB }, journal: journal("success"), prompts: 2 }]; + case "resume": + return [ + { document: join(SCENARIOS, "resume-seed.md"), props: { ...base, marker: markers.r }, journal: journal("resume-seed"), prompts: 1 }, + { document: join(SCENARIOS, "resume-continuation.md"), props: base, journal: journal("resume-continuation"), prompts: 1 }, + ]; + case "failure": + return [{ document: join(SCENARIOS, "failure.md"), props: { ...base, denied: deniedFile }, journal: journal("failure"), prompts: 1, denyAll: true }]; + case "cancel": + return [{ document: join(SCENARIOS, "cancel.md"), props: { ...base, first: fileA, second: fileB }, journal: journal("cancel"), prompts: 1, cancelOn: fileA }]; + case "subagent": + return [{ document: join(SCENARIOS, "subagent.md"), props: { ...base, first: fileA }, journal: journal("subagent"), prompts: 1 }]; + } + })(); + const submissions = plan.reduce((sum, invocation) => sum + invocation.prompts, 0); + if (submissions > ALLOWANCE[selection.scenario]) { + throw new Error("the scenario plans more outer Prompts than its allowance"); + } + + const start = Date.now(); + const admissionAt = start + ADMISSION_SECONDS * 1000; + const measured = yield* measure(selection, fixture); + const joins: Joined[] = []; + let admitted = 0; + let stopped: string | undefined; + for (const [index, invocation] of plan.entries()) { + if (Date.now() > admissionAt) { + stopped = "admission-deadline"; + break; + } + admitted += invocation.prompts; + const joined = yield* invoke(selection, invocation, fixture, admissionAt, join(support, `child-${index}-stderr.txt`)); + joins.push(joined); + if (joined.code === undefined && joined.signal !== undefined && joined.signal !== "SIGINT") { + stopped = "cleanup-failed"; + break; + } + } + + const acknowledged: Acknowledged[] = []; + for (const invocation of plan) { + acknowledged.push(...((yield* readAcknowledged(invocation.journal)) ?? [])); + } + const firstNative = yield* nativeFor(selection.agent, acknowledged[0]); + const prompts: PromptReport[] = []; + const supportRecords: unknown[] = []; + for (const [index, entry] of acknowledged.entries()) { + const usage = entry.record.usage; + const native = selection.scenario === "resume" ? (yield* nativeFor(selection.agent, entry))?.prompts.at(index) : firstNative?.prompts[index]; + const acpx = yield* acpxAgreement(join(homedir(), ".acpx"), usage); + prompts.push({ + status: entry.record.status, + usage: shareable(usage), + checks: { + usage: usage === undefined ? check("unavailable", "usage-unavailable") : check("verified", "usage-retained"), + association: acpx.association, + persistedAgreement: acpx.agreement, + independentNative: nativeComparison(selection.agent, usage, native), + }, + native: native === undefined ? undefined : { calls: native.calls, toolCalls: native.toolCalls, subagentCalls: native.subagentCalls }, + }); + supportRecords.push({ + name: entry.name, + sessionKey: entry.record.sessionKey, + agentSessionId: entry.record.agentSessionId, + usage, + native: native ?? null, + nativeSource: firstNative?.source ?? null, + }); + } + yield* writeTextFile(join(support, "association.json"), `${JSON.stringify({ measured, prompts: supportRecords }, null, 2)}\n`); + + const scenario = scenarioChecks(selection, acknowledged, joins, firstNative, markers, yield* exists(deniedFile)); + const teardown = stopped === "cleanup-failed" + ? check("failed", "owned-cleanup-failed") + : joins.length === plan.length + ? check("verified", "owner-joined") + : check("not_run", stopped ?? "not-admitted"); + return { + agent: selection.agent, + host: selection.host, + scenario: selection.scenario, + allowance: ALLOWANCE[selection.scenario], + submittedPrompts: admitted, + acknowledgedPrompts: acknowledged.length, + provenance: { + xmd: { + ...(measured.xmdVersion === undefined ? {} : { version: measured.xmdVersion }), + ...(measured.xmdDigest === undefined ? {} : { digest: measured.xmdDigest }), + ...(measured.sourceRevision === undefined ? {} : { sourceRevision: measured.sourceRevision }), + ...(measured.sourceClean === undefined ? {} : { sourceClean: measured.sourceClean }), + }, + runtime: acknowledged[0]?.record.usage?.provenance.runtime ?? {}, + adapter: acknowledged[0]?.record.usage?.provenance.adapter ?? {}, + native: { + ...(acknowledged[0]?.record.usage?.provenance.native ?? {}), + ...(measured.nativeVersion === undefined ? {} : { observedVersion: measured.nativeVersion }), + }, + models: [...new Set(acknowledged.flatMap((entry) => entry.record.usage?.readings.flatMap((reading) => reading.model !== undefined && /^[A-Za-z0-9._:\-[\]]{1,80}$/u.test(reading.model) ? [reading.model] : []) ?? []))], + }, + prompts, + scenarioChecks: scenario, + readback: acknowledged.length > 0 ? check("verified", "canonical-journal-read-back") : check("unavailable", "no-acknowledged-history"), + teardown, + }; +} + +function scenarioChecks( + selection: Selection, + acknowledged: Acknowledged[], + joins: Joined[], + native: NativeSession | undefined, + markers: { a: string; b: string; r: string }, + deniedWritten: boolean, +): Record { + const completed = acknowledged.filter((entry) => entry.record.status === "completed"); + switch (selection.scenario) { + case "success": { + const sameSession = + acknowledged.length === 2 && + acknowledged[0].record.sessionKey === acknowledged[1].record.sessionKey && + acknowledged[0].record.agentSessionId !== undefined && + acknowledged[0].record.agentSessionId === acknowledged[1].record.agentSessionId; + const reads = native?.prompts[0]?.toolCalls; + return { + twoPromptsOneSession: completed.length === 2 && sameSession + ? check("verified", "two-acknowledged-prompts-one-session") + : check("failed", "two-prompts-not-observed"), + multicall: reads === undefined + ? check("unavailable", "native-tool-evidence-unavailable") + : reads >= 2 + ? check("verified", "two-or-more-native-tool-calls") + : check("failed", "fewer-than-two-native-tool-calls"), + recall: acknowledged[1]?.record.text.includes(markers.a) && acknowledged[1]?.record.text.includes(markers.b) + ? check("verified", "markers-recalled") + : check("limited", "markers-not-recalled"), + }; + } + case "resume": { + const [seed, continuation] = acknowledged; + const continued = + seed !== undefined && + continuation !== undefined && + seed.record.agentSessionId !== undefined && + continuation.record.agentSessionId === seed.record.agentSessionId; + return { + separateInvocations: joins.length === 2 ? check("verified", "two-joined-invocations") : check("failed", "separate-invocation-not-observed"), + exactTarget: continued && continuation?.record.status === "completed" + ? check("verified", "exact-retained-target-continued") + : continuation !== undefined && continuation.record.status !== "completed" + ? check("limited", "resume-refused-no-substitute") + : check("failed", "exact-target-not-observed"), + recall: continuation?.record.text.includes(markers.r) === true ? check("verified", "marker-recalled") : check("limited", "marker-not-recalled"), + }; + } + case "failure": { + const refused = acknowledged[0]?.record.permissions?.some((permission) => + permission.outcome.outcome === "cancelled" || + permission.options.some((option) => option.optionId === (permission.outcome.outcome === "selected" ? permission.outcome.optionId : "") && option.kind.startsWith("reject")), + ); + return { + attemptedWriteRefused: refused === true + ? check("verified", "host-refused-attempted-native-write") + : check("failed", "native-write-not-attempted"), + nothingWritten: deniedWritten ? check("failed", "denied-file-written") : check("verified", "denied-file-absent"), + acknowledgement: acknowledged.length === 1 + ? check("verified", `acknowledged-${acknowledged[0].record.status}`) + : check("limited", "no-acknowledged-record"), + }; + } + case "cancel": { + const joined = joins[0]; + return { + cancelledWhileActive: joined?.cancelledAfterReady === true + ? check("verified", "cancelled-after-ready-marker") + : check("failed", "ready-marker-not-observed-while-active"), + acknowledgement: acknowledged.length === 1 + ? check("verified", `acknowledged-${acknowledged[0].record.status}-partial`) + : check("verified", "no-acknowledged-record"), + }; + } + case "subagent": { + const observed = native?.prompts[0]?.subagentCalls; + return { + nativeSubagent: observed === undefined + ? check("unavailable", "native-source-unavailable") + : observed > 0 + ? check("verified", "native-subagent-work-observed") + : check("limited", "native-subagent-not-observed"), + inclusion: observed !== undefined && observed > 0 + ? check("limited", "subagent-inclusion-unverified") + : check("not_run", "no-subagent-work"), + }; + } + } +} + +/** Every check a selected scenario requires, and whether any of them is missing. */ +function verdict(report: Report): "completed" | "limited" | "incomplete" { + const required = Object.values(report.scenarioChecks); + if (report.teardown.outcome !== "verified" || required.some((item) => item.outcome === "failed" || item.outcome === "not_run" && item.reason !== "no-subagent-work")) { + return "incomplete"; + } + if (required.some((item) => item.outcome === "unavailable") && report.scenario !== "subagent") { + return "incomplete"; + } + if (report.prompts.some((prompt) => prompt.checks.persistedAgreement.outcome === "failed")) { + return "incomplete"; + } + const accounting = report.prompts.flatMap((prompt) => Object.values(prompt.checks)); + const limited = [...required, ...accounting].some((item) => item.outcome !== "verified"); + return limited || report.prompts.some((prompt) => prompt.usage?.coverage.outerPrompt !== "verified") ? "limited" : "completed"; +} + +const COMPLETION = { + completed: (output: string) => `Calibration completed. Read ${output}/report.md.`, + limited: (output: string) => `Calibration completed with accounting limitations. Read ${output}/report.md.`, + incomplete: (output: string) => `Calibration incomplete. Read ${output}/report.md before attempting another run.`, +}; + +function* writeIndex(output: string): Operation { + const files: { path: string; sha256: string }[] = []; + function* visit(directory: string, prefix: string): Operation { + for (const name of (yield* readdir(directory)).sort()) { + const path = join(directory, name); + const relative = prefix === "" ? name : `${prefix}/${name}`; + if ((yield* stat(path)).isDirectory()) { + if (name !== "fixture") { + yield* visit(path, relative); + } + } else if (name !== "evidence-index.json") { + files.push({ path: relative, sha256: yield* digestOf(path) }); + } + } + } + yield* visit(output, ""); + yield* writeTextFile(join(output, "evidence-index.json"), `${JSON.stringify({ files }, null, 2)}\n`); +} + +/** + * Keep children's output to this program: a child's own chatter is not the + * report, and what the run measured is read from its journal and evidence. + */ +function* quietChildren(): Operation { + yield* Stdio.around( + { + *stdout() { + return; + }, + *stderr() { + return; + }, + }, + { at: "min" }, + ); +} + +export function* calibrate(args: readonly string[]): Operation { + const parsed = parseSelection(args); + if (parsed.kind === "help") { + console.log(HELP); + return 0; + } + if (parsed.kind === "refused") { + console.error(parsed.message); + return 1; + } + const selection = parsed.selection; + yield* quietChildren(); + const output = resolve(selection.output); + yield* ensureDir(dirname(output)); + try { + yield* until(mkdir(output)); + } catch { + console.error(Refusal.exists(output)); + return 1; + } + const report = yield* runScenario(selection, output); + const outcome = verdict(report); + try { + yield* writeTextFile(join(output, "report.json"), `${JSON.stringify({ ...report, result: outcome }, null, 2)}\n`); + yield* writeTextFile(join(output, "report.md"), renderMarkdown(report, outcome)); + yield* writeIndex(output); + } catch (error) { + console.error(`Calibration report could not be written: ${error instanceof Error ? error.message : String(error)}`); + return 1; + } + if (report.teardown.outcome === "failed") { + console.error(Refusal.cleanup); + } + console.log(COMPLETION[outcome](output)); + return outcome === "incomplete" ? 1 : 0; +} + +if (import.meta.main) { + await main(function* (args) { + const status = yield* calibrate(args); + if (status !== 0) { + yield* exit(status); + } + }); +} diff --git a/examples/agent-usage/evidence.ts b/examples/agent-usage/evidence.ts new file mode 100644 index 000000000..b0ac0c8e9 --- /dev/null +++ b/examples/agent-usage/evidence.ts @@ -0,0 +1,459 @@ +import { exists, readdir, readTextFile } from "@effectionx/fs"; +import type { Operation } from "effection"; +import { join } from "node:path"; +import { parseDurableEvent } from "@executablemd/durable-streams"; +import { AGENT_PROMPT, parsePromptRecord } from "@executablemd/core/host"; +import type { PromptRecord } from "@executablemd/core/host"; +import type { PromptUsage, ReportedTokens, UsageReading } from "@executablemd/core"; + +/** How one check came out. */ +export type Outcome = "verified" | "limited" | "unavailable" | "failed" | "not_run"; + +/** One labelled fact with a fixed categorical reason. */ +export interface Check { + readonly outcome: Outcome; + readonly reason: string; +} + +export function check(outcome: Outcome, reason: string): Check { + return { outcome, reason }; +} + +/** One canonical Prompt the history acknowledged, read back without a provider. */ +export interface Acknowledged { + readonly name: string; + readonly record: PromptRecord; +} + +/** + * Every acknowledged Prompt a child's canonical journal holds, in order. + * + * Read through Core's own record parser, so this is the same reader a replay + * uses; a record that does not read back is refused rather than skipped. + */ +export function* readAcknowledged(journal: string): Operation { + if (!(yield* exists(journal))) { + return undefined; + } + const prompts: Acknowledged[] = []; + for (const line of (yield* readTextFile(journal)).split("\n")) { + if (line.trim().length === 0) { + continue; + } + const parsed = parseDurableEvent(line); + if (!parsed.ok) { + throw new Error("a canonical journal line does not read back as a durable event"); + } + const event = parsed.value; + if (event.type !== "yield" || event.description.type !== AGENT_PROMPT) { + continue; + } + if (event.result.status !== "ok") { + continue; + } + const record = parsePromptRecord(event.result.value); + if (record === undefined) { + throw new Error("a canonical Prompt record does not read back"); + } + prompts.push({ name: event.description.name, record }); + } + return prompts; +} + +export function tokensOf( + usage: PromptUsage | undefined, + source: UsageReading["source"], +): ReportedTokens | null | undefined { + const reading = usage?.readings.find( + (candidate) => candidate.source === source && candidate.kind === "tokens", + ); + return reading?.kind === "tokens" ? reading.tokens : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function asRecord(value: unknown): Record | undefined { + return isRecord(value) ? value : undefined; +} + +function count(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +/** What ACPX saved for one exact request, compared with the canonical response view. */ +export interface AcpxAgreement { + readonly association: Check; + readonly agreement: Check; +} + +/** + * Compare a canonical value with the owning ACPX record and its generated user message. + * + * The record and message are located by the identities the canonical value + * carries, never by equal counts, order or time. Agreement here proves the + * runtime's transport and storage, not that a total covers the outer Prompt. + */ +export function* acpxAgreement( + acpxRoot: string, + usage: PromptUsage | undefined, +): Operation { + const recordId = usage?.providerRecordId; + const messageId = usage?.providerMessageId; + if (recordId === undefined || messageId === undefined) { + return { + association: check("unavailable", "identity-unavailable"), + agreement: check("not_run", "association-unavailable"), + }; + } + const path = join(acpxRoot, "sessions", `${encodeURIComponent(recordId)}.json`); + if (!(yield* exists(path))) { + return { + association: check("unavailable", "acpx-record-unavailable"), + agreement: check("not_run", "association-unavailable"), + }; + } + const saved = asRecord(JSON.parse(yield* readTextFile(path))); + const messages = Array.isArray(saved?.messages) ? saved.messages : []; + const owned = messages.some( + (message) => asRecord(asRecord(message)?.User)?.id === messageId, + ); + const perRequest = asRecord(saved?.request_token_usage); + const entry = asRecord(perRequest?.[messageId]); + if (!owned && entry === undefined) { + return { + association: check("failed", "message-not-in-record"), + agreement: check("not_run", "association-unavailable"), + }; + } + const association = check( + owned ? "verified" : "limited", + owned ? "generated-message-in-owning-record" : "message-trimmed-from-record", + ); + const response = tokensOf(usage, "response"); + if (entry === undefined || response === undefined || response === null) { + return { association, agreement: check("unavailable", "no-saved-or-reported-response") }; + } + const saidByAcpx: Record = { + inputTokens: count(entry.input_tokens), + outputTokens: count(entry.output_tokens), + cachedReadTokens: count(entry.cache_read_input_tokens), + cachedWriteTokens: count(entry.cache_creation_input_tokens), + thoughtTokens: count(entry.thought_tokens), + totalTokens: count(entry.total_tokens), + }; + const agrees = Object.entries(saidByAcpx).every( + ([field, value]) => value === Reflect.get(response, field), + ); + return { + association, + agreement: check(agrees ? "verified" : "failed", agrees ? "saved-copy-agrees" : "saved-copy-disagrees"), + }; +} + +/** The independent native account of one outer Prompt, as selected counters. */ +export interface NativePrompt { + /** Internal model calls the native source recorded for this Prompt. */ + readonly calls: number; + /** Native tool calls the main loop made. */ + readonly toolCalls: number; + /** The whole outer Prompt's counters, where the source can say. */ + readonly outer?: Record; + /** The final internal request's counters, where the source can say. */ + readonly last?: Record; + /** Counters native subagent work recorded separately. */ + readonly subagent?: Record; + readonly subagentCalls: number; +} + +/** A native session's Prompts, located by its confirmed native identity, and where they came from. */ +export interface NativeSession { + readonly prompts: NativePrompt[]; + /** The exact private source file, for the private support record only. */ + readonly source: string; +} + +function add(into: Record, from: Record, keys: readonly string[]) { + for (const key of keys) { + const value = count(from[key]); + if (value !== undefined) { + into[key] = (into[key] ?? 0) + value; + } + } +} + +const CLAUDE_KEYS = [ + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", +] as const; + +function* findFile(root: string, accept: (name: string) => boolean, depth: number): Operation { + if (!(yield* exists(root))) { + return undefined; + } + for (const name of yield* readdir(root)) { + const path = join(root, name); + if (accept(name)) { + return path; + } + if (depth > 0 && !name.includes(".")) { + const found = yield* findFile(path, accept, depth - 1); + if (found !== undefined) { + return found; + } + } + } + return undefined; +} + +function lines(text: string): Record[] { + return text.split("\n").flatMap((line) => { + if (line.trim().length === 0) { + return []; + } + try { + const parsed = asRecord(JSON.parse(line)); + return parsed === undefined ? [] : [parsed]; + } catch { + return []; + } + }); +} + +/** + * Claude Code's own transcript for one native session. + * + * A user entry that is a typed prompt rather than a tool result starts a + * Prompt. Assistant entries repeat one API message per content block, so usage + * counts once per message id. Sidechain entries and subagent transcripts are + * the native account of subagent work, kept apart from the main loop. + */ +export function* claudeNative(home: string, sessionId: string): Operation { + const projects = join(home, ".claude", "projects"); + const file = yield* findFile(projects, (name) => name === `${sessionId}.jsonl`, 1); + if (file === undefined) { + return undefined; + } + const prompts: ClaudePrompt[] = []; + for (const entry of lines(yield* readTextFile(file))) { + const message = asRecord(entry.message); + if (entry.type === "user" && entry.isSidechain !== true && entry.isMeta !== true) { + const content = message?.content; + const typed = + typeof content === "string" || + (Array.isArray(content) && + content.some((block) => asRecord(block)?.type === "text") && + !content.some((block) => asRecord(block)?.type === "tool_result")); + if (typed) { + prompts.push({ calls: 0, toolCalls: 0, outer: {}, subagent: {}, seen: new Set(), subagentCalls: 0 }); + } + continue; + } + const current = prompts.at(-1); + if (entry.type !== "assistant" || current === undefined || message === undefined) { + continue; + } + const id = typeof message.id === "string" ? message.id : undefined; + const usage = asRecord(message.usage); + const content = Array.isArray(message.content) ? message.content : []; + current.toolCalls += entry.isSidechain === true + ? 0 + : content.filter((block) => asRecord(block)?.type === "tool_use").length; + if (id === undefined || usage === undefined || current.seen.has(id)) { + continue; + } + current.seen.add(id); + if (entry.isSidechain === true) { + current.subagentCalls++; + add(current.subagent, usage, CLAUDE_KEYS); + } else { + current.calls++; + add(current.outer, usage, CLAUDE_KEYS); + } + } + return { + source: file, + prompts: prompts.map(({ calls, toolCalls, outer, subagent, subagentCalls }) => ({ + calls, + toolCalls, + outer, + ...(subagentCalls > 0 ? { subagent } : {}), + subagentCalls, + })), + }; +} + +/** One Codex Prompt while its rollout is being read. */ +interface CodexPrompt { + calls: number; + toolCalls: number; + start: Record; + end: Record; + last?: Record; +} + +/** One Claude Prompt while its transcript is being read. */ +interface ClaudePrompt { + calls: number; + toolCalls: number; + outer: Record; + subagent: Record; + subagentCalls: number; + readonly seen: Set; +} + +const CODEX_KEYS = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", +] as const; + +/** + * Codex's own rollout for one native thread. + * + * Each `token_count` event carries both the running thread total and the last + * request's reading. A started task starts a Prompt; the difference between the + * running totals at its end and at the previous Prompt's end is the outer + * Prompt's own total, and the last reading in it is its final request. + */ +export function* codexNative(home: string, threadId: string): Operation { + const sessions = join(home, ".codex", "sessions"); + const file = yield* findFile(sessions, (name) => name.endsWith(`${threadId}.jsonl`), 3); + if (file === undefined) { + return undefined; + } + let previousTotal: Record = {}; + let runningTotal: Record = {}; + const prompts: CodexPrompt[] = []; + const entries = lines(yield* readTextFile(file)); + // A turn starts where the rollout says a task started; older rollouts mark + // only the user message. + const boundary = entries.some( + (entry) => entry.type === "event_msg" && asRecord(entry.payload)?.type === "task_started", + ) + ? "task_started" + : "user_message"; + for (const entry of entries) { + const payload = asRecord(entry.payload); + if (entry.type === "event_msg" && payload?.type === boundary) { + previousTotal = { ...runningTotal }; + prompts.push({ calls: 0, toolCalls: 0, start: previousTotal, end: previousTotal }); + continue; + } + const current = prompts.at(-1); + if (current === undefined) { + continue; + } + if (entry.type === "response_item" && typeof payload?.type === "string" && /call$/u.test(payload.type) && !/output$/u.test(payload.type)) { + current.toolCalls++; + } + if (entry.type === "event_msg" && payload?.type === "token_count") { + const info = asRecord(payload.info); + const total = asRecord(info?.total_token_usage); + const last = asRecord(info?.last_token_usage); + if (total !== undefined) { + const next: Record = {}; + add(next, total, CODEX_KEYS); + if (JSON.stringify(next) !== JSON.stringify(runningTotal)) { + current.calls++; + } + runningTotal = next; + } + if (last !== undefined) { + const reading: Record = {}; + add(reading, last, CODEX_KEYS); + current.last = reading; + } + } + current.end = { ...runningTotal }; + } + return { + source: file, + prompts: prompts.map((prompt) => { + const outer: Record = {}; + for (const key of CODEX_KEYS) { + const end = prompt.end[key]; + if (end !== undefined) { + outer[key] = end - (prompt.start[key] ?? 0); + } + } + return { + calls: prompt.calls, + toolCalls: prompt.toolCalls, + outer, + ...(prompt.last === undefined ? {} : { last: prompt.last }), + subagentCalls: 0, + }; + }), + }; +} + +/** The canonical response view, spelled in a native source's own counter names. */ +function canonicalAs(agent: "claude" | "codex", tokens: ReportedTokens): Record { + const named: Record = {}; + const put = (key: string, value: number | undefined) => { + if (value !== undefined) { + named[key] = value; + } + }; + if (agent === "claude") { + put("input_tokens", tokens.inputTokens); + put("output_tokens", tokens.outputTokens); + put("cache_read_input_tokens", tokens.cachedReadTokens); + put("cache_creation_input_tokens", tokens.cachedWriteTokens); + return named; + } + // The Codex adapter subtracts cached input from native input before replying. + put( + "input_tokens", + tokens.inputTokens === undefined ? undefined : tokens.inputTokens + (tokens.cachedReadTokens ?? 0), + ); + put("cached_input_tokens", tokens.cachedReadTokens); + put("output_tokens", tokens.outputTokens); + put("reasoning_output_tokens", tokens.thoughtTokens); + put("total_tokens", tokens.totalTokens); + return named; +} + +function sameOn(left: Record, right: Record | undefined): boolean { + if (right === undefined) { + return false; + } + const keys = Object.keys(left); + return keys.length > 0 && keys.every((key) => left[key] === right[key]); +} + +/** + * Compare one acknowledged response view with the native account of the same Prompt. + * + * `limited` when the canonical value matches only the final internal request + * of a Prompt that made more than one: a bound, not a total. A disagreement is + * a documented limitation, never a reason to manufacture a total. + */ +export function nativeComparison( + agent: "claude" | "codex", + usage: PromptUsage | undefined, + native: NativePrompt | undefined, +): Check { + if (native === undefined) { + return check("unavailable", "native-source-unavailable"); + } + const response = tokensOf(usage, "response"); + if (response === undefined || response === null) { + return check("unavailable", "no-reported-response"); + } + const canonical = canonicalAs(agent, response); + if (sameOn(canonical, native.outer)) { + return check("verified", native.calls > 1 ? "matches-native-outer-total-multicall" : "matches-native-outer-total"); + } + if (sameOn(canonical, native.last)) { + return check("limited", "final-request-only"); + } + return check("limited", "native-disagreement"); +} diff --git a/examples/agent-usage/report.ts b/examples/agent-usage/report.ts new file mode 100644 index 000000000..5226015b8 --- /dev/null +++ b/examples/agent-usage/report.ts @@ -0,0 +1,132 @@ +import type { UsageCoverage, UsageReading } from "@executablemd/core"; +import type { Check } from "./evidence.ts"; + +/** One acknowledged Prompt, as the shareable report states it. */ +export interface PromptReport { + readonly status: "completed" | "failed" | "cancelled"; + readonly usage?: { + readonly readings: readonly UsageReading[]; + readonly coverage: UsageCoverage; + }; + readonly checks: { + readonly usage: Check; + readonly association: Check; + readonly persistedAgreement: Check; + readonly independentNative: Check; + }; + readonly native?: { readonly calls: number; readonly toolCalls: number; readonly subagentCalls: number }; +} + +/** + * The feature-local calibration result. + * + * Selected counts, strict version tokens and digests, reviewed model + * identifiers, fixed source semantics and categorical checks. No identity, + * path, marker, prompt or reply text, and no native log, reaches it. + */ +export interface Report { + readonly agent: "claude" | "codex"; + readonly host: "source" | "compiled"; + readonly scenario: string; + readonly allowance: number; + readonly submittedPrompts: number; + readonly acknowledgedPrompts: number; + readonly provenance: { + readonly xmd: { + readonly version?: string; + readonly digest?: string; + readonly sourceRevision?: string; + readonly sourceClean?: boolean; + }; + readonly runtime: { readonly version?: string; readonly digest?: string }; + readonly adapter: { readonly version?: string; readonly digest?: string }; + readonly native: { readonly version?: string; readonly digest?: string; readonly observedVersion?: string }; + readonly models: readonly string[]; + }; + readonly prompts: readonly PromptReport[]; + readonly scenarioChecks: Readonly>; + readonly readback: Check; + readonly teardown: Check; +} + +function line(name: string, item: Check): string { + return `| ${name} | ${item.outcome} | ${item.reason} |`; +} + +function value(text: string | undefined): string { + return text ?? "unknown"; +} + +function tokens(reading: UsageReading): string { + if (reading.kind === "tokens") { + if (reading.tokens === null) { + return "null"; + } + const entries = Object.entries(reading.tokens); + return entries.length === 0 ? "none reported" : entries.map(([key, count]) => `${key}=${count}`).join(", "); + } + if (reading.kind === "context") { + return `used=${reading.used ?? "unknown"}, size=${reading.size ?? "unknown"}`; + } + return `amount=${reading.amount ?? "unknown"}, currency=${reading.currency ?? "unknown"}`; +} + +export function renderMarkdown(report: Report, result: string): string { + const out: string[] = [ + `# Usage calibration: ${report.agent} on ${report.host}, ${report.scenario}`, + "", + `Result: **${result}**. Allowance ${report.allowance} outer Prompts; submitted ${report.submittedPrompts}; acknowledged ${report.acknowledgedPrompts}.`, + "", + "## Builds", + "", + `- XMD: version ${value(report.provenance.xmd.version)}, digest ${value(report.provenance.xmd.digest)}, source revision ${value(report.provenance.xmd.sourceRevision)}${report.provenance.xmd.sourceClean === false ? " (uncommitted changes present)" : ""}`, + `- Runtime: version ${value(report.provenance.runtime.version)}`, + `- Adapter: version ${value(report.provenance.adapter.version)}, digest ${value(report.provenance.adapter.digest)}`, + `- Native: version ${value(report.provenance.native.version)}, digest ${value(report.provenance.native.digest)}, observed ${value(report.provenance.native.observedVersion)}`, + `- Reported models: ${report.provenance.models.length === 0 ? "unknown" : report.provenance.models.join(", ")}`, + "", + "## Scenario", + "", + "| Check | Outcome | Reason |", + "| --- | --- | --- |", + ...Object.entries(report.scenarioChecks).map(([name, item]) => line(name, item)), + line("history readback", report.readback), + line("joined teardown", report.teardown), + "", + ]; + for (const [index, prompt] of report.prompts.entries()) { + out.push(`## Prompt ${index + 1} (${prompt.status})`, ""); + out.push("| Check | Outcome | Reason |", "| --- | --- | --- |"); + out.push(line("usage retained", prompt.checks.usage)); + out.push(line("request to provider message", prompt.checks.association)); + out.push(line("persisted ACPX agreement", prompt.checks.persistedAgreement)); + out.push(line("independent native comparison", prompt.checks.independentNative)); + out.push(""); + if (prompt.native !== undefined) { + out.push( + `Native account: ${prompt.native.calls} model calls, ${prompt.native.toolCalls} tool calls, ${prompt.native.subagentCalls} subagent calls.`, + "", + ); + } + if (prompt.usage === undefined) { + out.push("No usage was retained for this Prompt; its consumption is unknown.", ""); + continue; + } + out.push( + `Coverage: final report ${prompt.usage.coverage.finalReport}; outer Prompt ${prompt.usage.coverage.outerPrompt}; limitations ${prompt.usage.coverage.limitations.join(", ") || "none"}.`, + "", + "| Source | Scope | Counter | Model | Values |", + "| --- | --- | --- | --- | --- |", + ...prompt.usage.readings.map( + (reading) => + `| ${reading.source} | ${reading.scope} | ${reading.kind === "tokens" ? reading.counter : reading.kind} | ${reading.model ?? ""} | ${tokens(reading)} |`, + ), + "", + ); + } + out.push( + "Source views of the same work are not additional spend, and no total here is an estimate or a price.", + "", + ); + return out.join("\n"); +} diff --git a/examples/agent-usage/scenarios/cancel.md b/examples/agent-usage/scenarios/cancel.md new file mode 100644 index 000000000..932f9723e --- /dev/null +++ b/examples/agent-usage/scenarios/cancel.md @@ -0,0 +1,14 @@ +--- +props: + agent: { type: string } + session: { type: string } + first: { type: string } + second: { type: string } +--- + + + +First reply with the exact line USAGE_CALIBRATION_READY. Then read {props.first} and {props.second} with separate native file-read tool calls and explain their contents in detail. Do not change files or start subagents. + + + diff --git a/examples/agent-usage/scenarios/failure.md b/examples/agent-usage/scenarios/failure.md new file mode 100644 index 000000000..c2248c3e5 --- /dev/null +++ b/examples/agent-usage/scenarios/failure.md @@ -0,0 +1,13 @@ +--- +props: + agent: { type: string } + session: { type: string } + denied: { type: string } +--- + + + +Use your native file-write tool to write the text calibration into {props.denied}. Do not use a shell or another tool. If permission is denied, do not try another method. + + + diff --git a/examples/agent-usage/scenarios/resume-continuation.md b/examples/agent-usage/scenarios/resume-continuation.md new file mode 100644 index 000000000..ba312618d --- /dev/null +++ b/examples/agent-usage/scenarios/resume-continuation.md @@ -0,0 +1,12 @@ +--- +props: + agent: { type: string } + session: { type: string } +--- + + + +Without tools, repeat the harmless calibration marker I gave you in our previous request. + + + diff --git a/examples/agent-usage/scenarios/resume-seed.md b/examples/agent-usage/scenarios/resume-seed.md new file mode 100644 index 000000000..791fc7d84 --- /dev/null +++ b/examples/agent-usage/scenarios/resume-seed.md @@ -0,0 +1,13 @@ +--- +props: + agent: { type: string } + session: { type: string } + marker: { type: string } +--- + + + +Remember this harmless calibration marker for our next request: {props.marker}. Reply with the marker. Use no tools. + + + diff --git a/examples/agent-usage/scenarios/subagent.md b/examples/agent-usage/scenarios/subagent.md new file mode 100644 index 000000000..3214001bb --- /dev/null +++ b/examples/agent-usage/scenarios/subagent.md @@ -0,0 +1,13 @@ +--- +props: + agent: { type: string } + session: { type: string } + first: { type: string } +--- + + + +Ask one native subagent to read {props.first} and return its marker. Do not read the file yourself. If native subagents are unavailable, say unavailable and stop. Do not change files. + + + diff --git a/examples/agent-usage/scenarios/success.md b/examples/agent-usage/scenarios/success.md new file mode 100644 index 000000000..0086301c9 --- /dev/null +++ b/examples/agent-usage/scenarios/success.md @@ -0,0 +1,15 @@ +--- +props: + agent: { type: string } + session: { type: string } + first: { type: string } + second: { type: string } +--- + + + +Read {props.first} with one native file-read tool call. Then read {props.second} with a second native file-read tool call. Do not combine the reads in one command. Reply with the two markers you read. Do not change files or start subagents. +Without tools, repeat the two markers you read in the previous request. + + + diff --git a/examples/agent-usage/selection.ts b/examples/agent-usage/selection.ts new file mode 100644 index 000000000..7ee878772 --- /dev/null +++ b/examples/agent-usage/selection.ts @@ -0,0 +1,108 @@ +import { isAbsolute } from "node:path"; + +/** One selected calibration experiment. */ +export interface Selection { + readonly host: "source" | "compiled"; + readonly agent: "claude" | "codex"; + readonly scenario: Scenario; + readonly output: string; + /** The exact executable a compiled selection measures. */ + readonly xmd?: string; +} + +export type Scenario = "success" | "resume" | "failure" | "cancel" | "subagent"; + +/** What the command line asked for, before anything is created or started. */ +export type Parsed = + | { readonly kind: "help" } + | { readonly kind: "refused"; readonly message: string } + | { readonly kind: "selected"; readonly selection: Selection }; + +export const HELP = `Measure one bundled Agent through one production XMD host. Select exactly one scenario. This run may spend tokens; --allow-paid is required. The output directory must be new. Missing measurements remain unknown. + + --host=source|compiled + --agent=claude|codex + --scenario=success|resume|failure|cancel|subagent + --output= + --xmd= required only for compiled + --allow-paid required for any live scenario + --help prints help and starts no Agent +`; + +export const Refusal = Object.freeze({ + paid: "Calibration requires --allow-paid; no Agent was started.", + compiled: "--host=compiled requires --xmd with an absolute executable path to measure.", + unsupported: + "Unsupported calibration selection. Choose a documented host, bundled Agent and scenario; see --help.", + cleanup: + "Calibration stopped because owned cleanup did not complete successfully. No later work was started.", + exists: (path: string) => + `Calibration output already exists: ${path}. Choose a new --output directory.`, +}); + +/** The maximum outer Prompts each scenario may submit. */ +export const ALLOWANCE: Readonly> = Object.freeze({ + success: 2, + resume: 2, + failure: 1, + cancel: 1, + subagent: 1, +}); + +const HOSTS = ["source", "compiled"] as const; +const AGENTS = ["claude", "codex"] as const; +const SCENARIOS = ["success", "resume", "failure", "cancel", "subagent"] as const; +const VALUED = ["host", "agent", "scenario", "output", "xmd"] as const; + +function member(values: readonly T[], value: string | undefined): T | undefined { + return values.find((candidate) => candidate === value); +} + +/** Read the grammar. Every refusal happens here, before any output or provider work. */ +export function parseSelection(args: readonly string[]): Parsed { + if (args.includes("--help")) { + return { kind: "help" }; + } + const values = new Map(); + let paid = false; + for (const arg of args) { + if (arg === "--allow-paid") { + if (paid) { + return { kind: "refused", message: Refusal.unsupported }; + } + paid = true; + continue; + } + const match = /^--([a-z]+)=(.*)$/u.exec(arg); + const name = match?.[1]; + const value = match?.[2]; + if (name === undefined || value === undefined || value.length === 0) { + return { kind: "refused", message: Refusal.unsupported }; + } + if (member(VALUED, name) === undefined || values.has(name)) { + return { kind: "refused", message: Refusal.unsupported }; + } + values.set(name, value); + } + const host = member(HOSTS, values.get("host")); + const agent = member(AGENTS, values.get("agent")); + const scenario = member(SCENARIOS, values.get("scenario")); + const output = values.get("output"); + const xmd = values.get("xmd"); + if (host === undefined || agent === undefined || scenario === undefined || output === undefined) { + return { kind: "refused", message: Refusal.unsupported }; + } + if (host === "source" && xmd !== undefined) { + return { kind: "refused", message: Refusal.unsupported }; + } + if (host === "compiled" && (xmd === undefined || !isAbsolute(xmd))) { + return { kind: "refused", message: Refusal.compiled }; + } + if (!paid) { + return { kind: "refused", message: Refusal.paid }; + } + return { + kind: "selected", + selection: { host, agent, scenario, output, ...(xmd === undefined ? {} : { xmd }) }, + }; +} diff --git a/packages/acp/mod.ts b/packages/acp/mod.ts index 3f696b0fd..44d2499e1 100644 --- a/packages/acp/mod.ts +++ b/packages/acp/mod.ts @@ -29,6 +29,7 @@ export type { AcpxSessionPolicy, SessionRouteContext, } from "./src/provider.ts"; +export type { PromptUsageHost } from "./src/usage.ts"; /** The agent ACPX selects when nothing else is configured. */ export { DEFAULT_AGENT_NAME } from "./src/acpx-runtime.ts"; export { createAcpRuntime, createAgentRegistry } from "./src/acpx-runtime.ts"; diff --git a/packages/acp/src/adapter-snapshots.ts b/packages/acp/src/adapter-snapshots.ts index 2f8fad408..f3b929d1e 100644 --- a/packages/acp/src/adapter-snapshots.ts +++ b/packages/acp/src/adapter-snapshots.ts @@ -713,9 +713,27 @@ export function carriesEmbeddedAdapter(adapters: EmbeddedAdapters, agentName: st */ export function embeddedAdapterDependencies( adapters: EmbeddedAdapters, -): Pick { +): Pick< + AcpxProviderDependencies, + "agentRegistry" | "prepareAgent" | "agentProcessEnv" | "adapterBuild" +> { return { agentRegistry: overlaidAdapterRegistry(adapters), + // The snapshot a turn's usage names: only for a command that is this + // host's own carried adapter, so a registry entry that merely shares the + // agent's name is never described as a build this host verified. + adapterBuild(agentName, agentCommand) { + if ( + !carriesEmbeddedAdapter(adapters, agentName) || + agentCommand !== adapters.command(agentName) + ) { + return undefined; + } + const snapshot = EMBEDDED_ADAPTER_SNAPSHOTS.find((entry) => entry.provider === agentName); + return snapshot === undefined + ? undefined + : { version: snapshot.version, digest: `sha256:${snapshot.sha256}` }; + }, *agentProcessEnv(agentName, agentCommand) { if ( agentName !== "claude" || diff --git a/packages/acp/src/events.ts b/packages/acp/src/events.ts index 420c5e2d4..159bef825 100644 --- a/packages/acp/src/events.ts +++ b/packages/acp/src/events.ts @@ -14,6 +14,12 @@ * as a property on the event, which every holder of the event could read and * every builder of one could write. A turn that was cancelled, failed, or * refused by the host names nothing. + * + * Usage is read on the same pass, by the same reader: a live update is offered + * to the turn's collector as it arrives, and the terminal result — whichever + * way the turn ended — settles it before the terminal event is sent. The + * collector never throws, and it is consulted outside the failure handling + * below, so observing a turn cannot cancel, close or fail it. */ import { each, race, stream, until } from "effection"; @@ -23,6 +29,7 @@ import type { AcpRuntimeTurn, AcpRuntimeTurnResult } from "./acpx-runtime.ts"; import { checkpointFromResult } from "./checkpoint.ts"; import { XmdResponse, XMD_RESPONSE, xmdPromptEvent } from "./xmd-response.ts"; import type { Registration } from "./permission-bridge.ts"; +import type { TurnUsage } from "./usage.ts"; export interface TurnIdentity { agent: string; @@ -56,14 +63,17 @@ export function* consumeTurn( checkpoint?: TurnCheckpoint, inputs?: Registration, closeFailed?: () => Operation, + usage?: TurnUsage, ): Operation { yield* channel.send({ type: "started", agent: identity.agent, session: identity.session }); let text = ""; let terminal: AgentPromptEvent; let named: AgentPromptCheckpoint | undefined; + let settled: AcpRuntimeTurnResult | undefined; try { const reading = function* (): Operation { for (const event of yield* each(stream(turn.events))) { + usage?.observe(event); if (event.type === "text_delta" && (event.stream ?? "output") === "output") { text += event.text; yield* channel.send({ type: "text_delta", text: event.text }); @@ -100,6 +110,7 @@ export function* consumeTurn( ]); } const result = yield* until(turn.result); + settled = result; terminal = mapResult(result, turn.requestId); // Read from the result rather than from the mapped event: a turn ACP // reported as completed under a stop reason this host treats as a failure @@ -112,7 +123,9 @@ export function* consumeTurn( const cleanup: Error[] = []; for (const finish of [ () => until(turn.cancel()), - () => until(turn.result), + function* (): Operation { + settled = yield* until(turn.result); + }, ...(closeFailed === undefined ? [] : [closeFailed]), ]) { try { @@ -140,6 +153,7 @@ export function* consumeTurn( if (named !== undefined) { checkpoint?.(terminal, named); } + usage?.settle(settled); yield* channel.send(terminal); markCompleted(); yield* channel.close(text); diff --git a/packages/acp/src/provider.ts b/packages/acp/src/provider.ts index 27737ef89..dca8f85c4 100644 --- a/packages/acp/src/provider.ts +++ b/packages/acp/src/provider.ts @@ -76,6 +76,7 @@ import type { Session, SessionConfiguration, SessionLaunchResult, + UsageBuild, } from "@executablemd/core"; import { AcpConfigurationError, @@ -125,6 +126,8 @@ import { strictPermissions, } from "./permission-bridge.ts"; import { consumeTurn } from "./events.ts"; +import { capturePromptUsage, turnUsage } from "./usage.ts"; +import type { PromptUsageHost, TurnUsage } from "./usage.ts"; import { checkpointFromResult } from "./checkpoint.ts"; import { resolveSessionDetails } from "./session-key.ts"; import { useSerialQueues } from "./serial-queue.ts"; @@ -285,6 +288,16 @@ export interface AcpxProviderDependencies { * asks for one prepares nothing. */ prepareAgent?: (agentName: string) => Operation; + /** + * What this host states about the XMD build its Prompt usage is measured on. + * + * Supplied by the host that knows what it is, beside the adapters it carries. + * Absent means this host collects no Prompt usage: turns run exactly as they + * would have, and their records carry none. + */ + promptUsageHost?: PromptUsageHost; + /** The carried adapter build behind an agent's command, when this host carries it. */ + adapterBuild?: (agentName: string, agentCommand: string) => UsageBuild | undefined; /** Child-only host inputs for the selected command; never session configuration. */ agentProcessEnv?: ( agentName: string, @@ -1106,6 +1119,8 @@ function* useAcpxProviderState( const coordinator = dependencies?.coordinator; const routeStore = dependencies?.routeStore; const executableObserver = dependencies?.executableObserver; + const promptUsageHost = dependencies?.promptUsageHost; + const adapterBuild = dependencies?.adapterBuild; const agentCwd = dependencies?.agentCwd ?? cwd; const prepareAgent = dependencies?.prepareAgent; // Copied out of the host's containers once, here, and never read from them @@ -1921,6 +1936,16 @@ function* useAcpxProviderState( } /** The resolved agent command behind session details or an existing entry. */ + /** The adapter build a turn's usage names, when this host carries one for the command. */ + function adapterUsageBuild(agentName: string, agentCommand: string): { adapter?: UsageBuild } { + try { + const build = adapterBuild?.(agentName, agentCommand); + return build === undefined ? {} : { adapter: build }; + } catch { + return {}; + } + } + function agentCommandOf(prepared: Prepared): string { return prepared.kind === "existing" ? prepared.entry.agentCommand : prepared.agentCommand; } @@ -2514,6 +2539,7 @@ function* useAcpxProviderState( entry: ManagedSession, details: AcpxSessionDetails, turn: AcpRuntimeTurn, + usage?: TurnUsage, ): Operation { const accepted: Result = yield* until( turn.materialized.then( @@ -2532,6 +2558,9 @@ function* useAcpxProviderState( // barrier only knows that this session is still awaiting its first // accepted turn. const settled = yield* until(turn.result); + // Events were never read for this turn, so the result is all there is to + // report — and it is reported here, before the failure ends the turn. + usage?.settle(settled); if (settled.status === "failed") { throw new Error(settled.error.message); } @@ -2772,6 +2801,10 @@ function* useAcpxProviderState( ): Stream { return { *[Symbol.iterator]() { + // First, and once: this subscription's usage receiver, captured before + // any native work so the turn reports to the subscription that started + // it and to no later one. + const reportUsage = yield* capturePromptUsage(launchCoordinator, promptUsageHost); // Same distinction as a session resolution's: an ordinary turn resolves // its agent through the chain, and a conversation's turn was handed the // agent its own session was resolved under. @@ -2908,6 +2941,7 @@ function* useAcpxProviderState( const callbacks = agentName === "xmd" ? registration : undefined; const timeoutMs = options?.timeout; + const requestId = randomUUID(); // The runtime that made this handle, never "the" runtime: a turn // started through another partition would be a second child in a // conversation the first already owns. @@ -2915,7 +2949,7 @@ function* useAcpxProviderState( handle: entry.handle, text: content, mode: "prompt", - requestId: randomUUID(), + requestId, timeoutMs, ...(callbacks === undefined ? {} @@ -2940,6 +2974,23 @@ function* useAcpxProviderState( }), }); activeTurns.add(turn); + const usage: TurnUsage | undefined = + reportUsage === undefined || promptUsageHost === undefined + ? undefined + : turnUsage(reportUsage, { + agent: agentName, + requestId, + host: promptUsageHost, + ...adapterUsageBuild(agentName, agentCommandOf(prepared)), + ...(construction?.build === undefined + ? {} + : { + native: { + version: construction.build.binding.reportedVersion, + digest: `${construction.build.binding.executableDigest.algorithm}:${construction.build.binding.executableDigest.value}`, + }, + }), + }); let completed = false; yield* ensure(function* () { activeTurns.delete(turn); @@ -2971,7 +3022,7 @@ function* useAcpxProviderState( adapterFor(agentName)?.acceptance === "first-turn-acceptance" && prepared.kind === "details" ) { - yield* materialize(entry, prepared.details, turn); + yield* materialize(entry, prepared.details, turn, usage); } else { try { yield* until(turn.promptStarted); @@ -3013,6 +3064,7 @@ function* useAcpxProviderState( releasedHandle(entry); detachSessionHandle(entry.session.sessionKey, entry); }, + usage, ), ); return subscription; diff --git a/packages/acp/src/usage.ts b/packages/acp/src/usage.ts new file mode 100644 index 000000000..689a79a7e --- /dev/null +++ b/packages/acp/src/usage.ts @@ -0,0 +1,374 @@ +/** + * What one ACP turn reports about its own usage + * (specs/acp-client-spec.md §Prompt usage). + * + * The provider is the only reader of a turn's events and the only holder of its + * terminal result, so this is where a turn's readings are observed. It is also + * where they are interpreted: what a counter covers is a property of the + * adapter build that produced it, and that knowledge lives in the table below, + * one checked-in entry per adapter release whose sources were traced. A build + * the table does not name reports its readings with unknown scope and counter + * semantics rather than borrowing a neighbour's. + * + * Collection is observation. A reading that fails to normalize, a receiver that + * throws and a coordinator that offers no receiver all leave the turn exactly as + * it would have run, and never reach the provider-failure handling that cancels + * or closes a turn. + */ + +import type { Operation } from "effection"; +import type { + AgentLaunchCoordinator, + PromptUsage, + PromptUsageReporter, + ReportedTokens, + UsageBuild, + UsageProvenance, + UsageReading, +} from "@executablemd/core"; +import type { + AcpRuntimeEvent, + AcpRuntimeQuotaTokens, + AcpRuntimeTurnResult, + AcpRuntimeUsageBreakdown, +} from "./acpx-runtime.ts"; + +/** What a host states about the XMD build a turn runs under. */ +export interface PromptUsageHost { + readonly host: UsageProvenance["host"]; + readonly xmd: UsageProvenance["xmd"]; +} + +/** The ACP runtime this package executes; see `../vendor/acpx/PROVENANCE.md`. */ +const RUNTIME_BUILD: UsageBuild = Object.freeze({ version: "0.21.1" }); + +/** Fixed categorical limitations a record may carry. */ +export const UsageLimitation = Object.freeze({ + finalReportUnavailable: "final-report-unavailable", + sourceSemanticsUnknown: "source-semantics-unknown", + nativeBuildUnobserved: "native-build-unobserved", + adapterBuildUnknown: "adapter-build-unknown", + normalizationFailed: "normalization-failed", + updateBreakdownScopeUnknown: "update-breakdown-scope-unknown", + costScopeUnverified: "cost-scope-unverified", + claudeImputedZero: "claude-imputed-zero", + claudeQuotaOverlapsResponse: "claude-quota-model-overlaps-response", + claudeSubagentInclusionUnverified: "claude-subagent-inclusion-unverified", + codexLastRequestOnly: "codex-last-request-only", + codexReasoningInclusionUnverified: "codex-reasoning-inclusion-unverified", +}); + +interface ViewSemantics { + readonly scope: UsageReading["scope"]; + readonly counter: "turn-total" | "snapshot" | "increment" | "unknown"; +} + +/** What one traced adapter release's views mean. */ +interface SourceSemantics { + readonly response: ViewSemantics; + readonly quotaTotal: ViewSemantics; + readonly quotaModel: ViewSemantics; + /** How much of the outer Prompt a received final report covers. */ + readonly outerPrompt: "verified" | "partial" | "unknown"; + readonly limitations: readonly string[]; +} + +/** + * The adapter releases whose usage sources were traced, by provider and version. + * + * Claude 0.70.0 reports its main agent loop's accumulated usage in the response + * and mirrors it in `quota.token_count`; `quota.model_usage` counts that loop + * together with Task subagents, sidechains and internal calls, so the two + * overlap. It fills absent native counters with zero and always reports + * `reasoningOutputTokens: 0` because thinking is billed inside output. + * + * Codex 1.6.2 builds both its response usage and its quota from the last + * internal request's reading, which is reset when a prompt begins, so neither + * covers a prompt that made several model calls. Reasoning is reported beside + * output without established inclusion. + */ +const SEMANTICS: Readonly> = Object.freeze({ + "claude@0.70.0": { + response: { scope: "main-loop", counter: "turn-total" }, + quotaTotal: { scope: "main-loop", counter: "turn-total" }, + quotaModel: { scope: "broader-model-work", counter: "turn-total" }, + outerPrompt: "partial", + limitations: [ + UsageLimitation.claudeImputedZero, + UsageLimitation.claudeQuotaOverlapsResponse, + UsageLimitation.claudeSubagentInclusionUnverified, + ], + }, + "codex@1.6.2": { + response: { scope: "last-request", counter: "snapshot" }, + quotaTotal: { scope: "last-request", counter: "snapshot" }, + quotaModel: { scope: "last-request", counter: "snapshot" }, + outerPrompt: "partial", + limitations: [ + UsageLimitation.codexLastRequestOnly, + UsageLimitation.codexReasoningInclusionUnverified, + ], + }, +}); + +const UNKNOWN_SEMANTICS: SourceSemantics = Object.freeze({ + response: { scope: "unknown", counter: "unknown" }, + quotaTotal: { scope: "unknown", counter: "unknown" }, + quotaModel: { scope: "unknown", counter: "unknown" }, + outerPrompt: "unknown", + limitations: [UsageLimitation.sourceSemanticsUnknown], +}); + +function semanticsFor(agent: string, adapter: UsageBuild | undefined): SourceSemantics { + if (adapter?.version === undefined) { + return UNKNOWN_SEMANTICS; + } + return SEMANTICS[`${agent}@${adapter.version}`] ?? UNKNOWN_SEMANTICS; +} + +/** + * The receiver for this subscription, captured once and before native work. + * + * An acquisition that fails is a turn with no receiver, never a turn refused: + * observation is optional, and an installation that offers none still runs. + */ +export function* capturePromptUsage( + coordinator: AgentLaunchCoordinator | undefined, + host: PromptUsageHost | undefined, +): Operation { + if (coordinator?.promptUsage === undefined || host === undefined) { + return undefined; + } + try { + return yield* coordinator.promptUsage(); + } catch { + return undefined; + } +} + +function tokensFrom( + value: AcpRuntimeUsageBreakdown | AcpRuntimeQuotaTokens | null | undefined, + fields: readonly (keyof ReportedTokens)[], +): ReportedTokens | null | undefined { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + const tokens: Record = {}; + for (const field of fields) { + const count = Reflect.get(value, field); + if (typeof count === "number" && Number.isFinite(count) && count >= 0) { + tokens[field] = count; + } + } + return tokens; +} + +const RESPONSE_FIELDS: readonly (keyof ReportedTokens)[] = [ + "inputTokens", + "outputTokens", + "cachedReadTokens", + "cachedWriteTokens", + "thoughtTokens", + "totalTokens", +]; +const QUOTA_FIELDS: readonly (keyof ReportedTokens)[] = [ + "totalTokens", + "inputTokens", + "cachedInputTokens", + "cachedWriteTokens", + "outputTokens", + "reasoningOutputTokens", +]; + +function finite(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +/** + * One subscribed turn's usage, from its live events to its terminal result. + * + * Live updates replace rather than add: an adapter's update is the current + * snapshot, so the latest context, cost and breakdown are what the turn last + * said, and an occupancy-only update leaves an earlier cost or breakdown in + * place. Nothing from a previous turn is carried, because each subscription + * gets its own collector. + */ +export interface TurnUsage { + /** A live event, as the turn's one reader saw it. Never throws. */ + observe(event: AcpRuntimeEvent): void; + /** Report what the turn retained, once. Never throws. */ + settle(result: AcpRuntimeTurnResult | undefined): void; +} + +export interface TurnUsageFacts { + readonly agent: string; + readonly requestId: string; + readonly host: PromptUsageHost; + readonly adapter?: UsageBuild; + readonly native?: UsageBuild; +} + +export function turnUsage(reporter: PromptUsageReporter, facts: TurnUsageFacts): TurnUsage { + let context: { used?: number; size?: number } | undefined; + let cost: { amount?: number; currency?: string } | undefined; + let breakdown: ReportedTokens | undefined; + let settled = false; + + function provenance(): UsageProvenance { + return { + host: facts.host.host, + xmd: { ...facts.host.xmd }, + runtime: { ...RUNTIME_BUILD }, + adapter: { ...facts.adapter }, + ...(facts.native === undefined ? {} : { native: { ...facts.native } }), + }; + } + + function build(result: AcpRuntimeTurnResult | undefined): PromptUsage { + const semantics = semanticsFor(facts.agent, facts.adapter); + const retained = result?.usage; + const readings: UsageReading[] = []; + const response = tokensFrom(retained?.response, RESPONSE_FIELDS); + if (response !== undefined) { + readings.push({ + source: "response", + scope: semantics.response.scope, + kind: "tokens", + counter: semantics.response.counter, + tokens: response, + }); + } + const quotaTotal = tokensFrom(retained?.quota?.tokenCount, QUOTA_FIELDS); + if (quotaTotal !== undefined) { + readings.push({ + source: "quota-total", + scope: semantics.quotaTotal.scope, + kind: "tokens", + counter: semantics.quotaTotal.counter, + tokens: quotaTotal, + }); + } + for (const row of retained?.quota?.modelUsage ?? []) { + const tokens = tokensFrom(row.tokenCount, QUOTA_FIELDS); + if (tokens !== undefined && typeof row.model === "string" && row.model.length > 0) { + readings.push({ + source: "quota-model", + scope: semantics.quotaModel.scope, + model: row.model, + kind: "tokens", + counter: semantics.quotaModel.counter, + tokens, + }); + } + } + const limitations = [...semantics.limitations]; + if (breakdown !== undefined) { + readings.push({ + source: "update", + scope: "unknown", + kind: "tokens", + counter: "unknown", + tokens: breakdown, + }); + limitations.push(UsageLimitation.updateBreakdownScopeUnknown); + } + if (context !== undefined) { + readings.push({ source: "update", scope: "session", kind: "context", ...context }); + } + if (cost !== undefined) { + readings.push({ source: "update", scope: "unknown", kind: "cost", ...cost }); + limitations.push(UsageLimitation.costScopeUnverified); + } + const finalReport = response !== undefined || quotaTotal !== undefined; + if (!finalReport) { + limitations.push(UsageLimitation.finalReportUnavailable); + } + if (facts.adapter?.version === undefined) { + limitations.push(UsageLimitation.adapterBuildUnknown); + } + if (facts.native === undefined) { + limitations.push(UsageLimitation.nativeBuildUnobserved); + } + return { + requestId: facts.requestId, + ...(retained?.acpxRecordId === undefined ? {} : { providerRecordId: retained.acpxRecordId }), + ...(retained?.promptMessageId === undefined + ? {} + : { providerMessageId: retained.promptMessageId }), + ...(retained?.sessionId === undefined ? {} : { providerSessionId: retained.sessionId }), + provenance: provenance(), + readings, + coverage: { + finalReport: finalReport ? "received" : "unavailable", + outerPrompt: finalReport ? semantics.outerPrompt : "unknown", + limitations, + }, + }; + } + + return { + observe(event) { + if (settled || event.type !== "status" || event.tag !== "usage_update") { + return; + } + try { + const used = finite(event.used); + const size = finite(event.size); + if (used !== undefined || size !== undefined) { + context = { + ...(used === undefined ? {} : { used }), + ...(size === undefined ? {} : { size }), + }; + } + if (event.cost !== undefined) { + const amount = finite(event.cost.amount); + const currency = + typeof event.cost.currency === "string" && event.cost.currency.length > 0 + ? event.cost.currency + : undefined; + if (amount !== undefined || currency !== undefined) { + cost = { + ...(amount === undefined ? {} : { amount }), + ...(currency === undefined ? {} : { currency }), + }; + } + } + const tokens = tokensFrom(event.breakdown, RESPONSE_FIELDS); + if (tokens) { + breakdown = tokens; + } + } catch { + // A reading that does not normalize is a reading this turn lacks. + } + }, + settle(result) { + if (settled) { + return; + } + settled = true; + let usage: PromptUsage; + try { + usage = build(result); + } catch { + usage = { + requestId: facts.requestId, + provenance: provenance(), + readings: [], + coverage: { + finalReport: "unavailable", + outerPrompt: "unknown", + limitations: [UsageLimitation.normalizationFailed], + }, + }; + } + try { + reporter(usage); + } catch { + // The receiver is observation. Its failure is not this turn's. + } + }, + }; +} diff --git a/packages/acp/tests/prompt-usage.test.ts b/packages/acp/tests/prompt-usage.test.ts new file mode 100644 index 000000000..99f0227f6 --- /dev/null +++ b/packages/acp/tests/prompt-usage.test.ts @@ -0,0 +1,297 @@ +/** + * Tier RU — the vendored runtime's per-turn usage + * (packages/acp/vendor/acpx/PROVENANCE.md §Selected Prompt usage). + * + * These speak real ACP to a real child, through the real vendored runtime. The + * fake agent answers each prompt with the usage and `_meta.quota` its script + * names, so what is proven is that the counters an agent put on a response, and + * the identities the runtime itself generated for that turn, reach the turn's + * terminal result on every way it ends — and that they are the same identities + * the runtime saved. + */ +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { ensure, until } from "effection"; +import type { Operation } from "effection"; +import { ensureDir, rm, writeTextFile } from "@effectionx/fs"; +import { chmod } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import * as os from "node:os"; +import * as path from "node:path"; +import { createAcpRuntime } from "../src/acpx-runtime.ts"; +import type { + AcpRuntime, + AcpRuntimeHandle, + AcpRuntimeTurnResult, + AcpSessionRecord, + AcpSessionStore, +} from "../src/acpx-runtime.ts"; + +/** + * A fake ACP agent. `XMD_RU_SCRIPT` is a JSON array, one entry per prompt in + * order: `{ usage?, meta?, delayMs?, accept? }`. `accept: false` withholds the + * first-turn acceptance update a materializing session waits for. + */ +const AGENT_SOURCE = `#!/usr/bin/env node +const script = JSON.parse(process.env.XMD_RU_SCRIPT ?? "[]"); +let prompts = 0; + +function send(message) { + process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + "\\n"); +} + +let buffer = ""; +process.stdin.on("data", (chunk) => { + buffer += chunk.toString("utf8"); + let index = buffer.indexOf("\\n"); + while (index >= 0) { + const line = buffer.slice(0, index); + buffer = buffer.slice(index + 1); + index = buffer.indexOf("\\n"); + if (line.trim().length > 0) { + handle(JSON.parse(line)); + } + } +}); + +function handle(request) { + if (request.id === undefined) { + return; + } + if (request.method === "initialize") { + send({ + id: request.id, + result: { + protocolVersion: request.params?.protocolVersion ?? 1, + agentCapabilities: { loadSession: false, promptCapabilities: {} }, + authMethods: [], + }, + }); + return; + } + if (request.method === "session/new") { + send({ id: request.id, result: { sessionId: "ru-" + process.pid } }); + return; + } + if (request.method === "session/prompt") { + const step = script[prompts++] ?? {}; + const sessionId = request.params.sessionId; + if (step.accept !== false) { + send({ + method: "session/update", + params: { + sessionId, + update: { + sessionUpdate: "session_info_update", + _meta: { "executablemd.session-materialization/v1": { state: "accepted" } }, + }, + }, + }); + } + send({ + method: "session/update", + params: { + sessionId, + update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "ok" } }, + }, + }); + const result = { stopReason: "end_turn" }; + if (step.usage !== undefined) { + result.usage = step.usage; + } + if (step.meta !== undefined) { + result._meta = step.meta; + } + setTimeout(() => send({ id: request.id, result }), step.delayMs ?? 0); + return; + } + send({ id: request.id, error: { code: -32601, message: "not implemented: " + request.method } }); +} +`; + +interface World { + dir: string; + agent: string; + records: Map; + store: AcpSessionStore; + /** Refuse any save of a record that already holds request usage. */ + failUsageSaves: boolean; +} + +function* useWorld(): Operation { + const dir = path.join(os.tmpdir(), `xmd-ru-${randomUUID()}`); + yield* ensureDir(dir); + yield* ensure(() => rm(dir, { recursive: true, force: true })); + const agent = path.join(dir, "fake-agent"); + yield* writeTextFile(agent, AGENT_SOURCE); + yield* until(chmod(agent, 0o755)); + const records = new Map(); + const world: World = { + dir, + agent, + records, + failUsageSaves: false, + store: { + load: (id) => Promise.resolve(structuredClone(records.get(id))), + save: (record) => { + const usage = Reflect.get(record, "request_token_usage"); + if (world.failUsageSaves && usage && Object.keys(usage).length > 0) { + return Promise.reject(new Error("injected finalization save failure")); + } + records.set(record.acpxRecordId, structuredClone(record)); + return Promise.resolve(); + }, + }, + }; + return world; +} + +function runtimeFor(world: World, script: unknown[]): AcpRuntime { + return createAcpRuntime({ + cwd: world.dir, + sessionStore: world.store, + agentRegistry: { resolve: () => world.agent, list: () => ["fake"] }, + permissionMode: "deny-all", + nonInteractivePermissions: "deny", + agentProcessEnv: { XMD_RU_SCRIPT: JSON.stringify(script) }, + }); +} + +function* ensureHandle( + world: World, + runtime: AcpRuntime, + sessionKey: string, + materialization?: "first-turn-acceptance", +): Operation { + return yield* until( + runtime.ensureSession({ + sessionKey, + agent: "fake", + mode: "persistent", + cwd: world.dir, + ...(materialization === undefined ? {} : { materialization }), + }), + ); +} + +function* turnOf( + runtime: AcpRuntime, + handle: AcpRuntimeHandle, + text: string, + timeoutMs = 20_000, +): Operation<{ requestId: string; result: AcpRuntimeTurnResult; materialized: boolean }> { + const requestId = randomUUID(); + const turn = runtime.startTurn({ handle, text, mode: "prompt", requestId, timeoutMs }); + const materialized = turn.materialized.then( + () => true, + () => false, + ); + const result = yield* until(turn.result); + return { requestId, result, materialized: yield* until(materialized) }; +} + +const EQUAL = { inputTokens: 7, outputTokens: 3, totalTokens: 10 }; +const QUOTA = { + token_count: { totalTokens: 10, inputTokens: 7, cachedInputTokens: 0, outputTokens: 3 }, + model_usage: [{ model: "fixture-model", token_count: { totalTokens: 12, outputTokens: 4 } }], +}; + +describe( + "Tier RU — vendored runtime usage", + { sanitizeOps: false, sanitizeResources: false }, + () => { + it("RU1: two turns with equal counters keep their own generated message and saved entry", function* () { + const world = yield* useWorld(); + const runtime = runtimeFor(world, [ + { usage: EQUAL, meta: { quota: QUOTA } }, + { usage: EQUAL, meta: { quota: QUOTA } }, + ]); + const handle = yield* ensureHandle(world, runtime, "ru-two"); + const first = yield* turnOf(runtime, handle, "alpha"); + const second = yield* turnOf(runtime, handle, "beta"); + yield* until(runtime.close({ handle, reason: "test" })); + + expect(first.result.status).toBe("completed"); + expect(second.result.status).toBe("completed"); + const one = first.result.usage; + const two = second.result.usage; + expect(one?.response).toEqual(EQUAL); + expect(two?.response).toEqual(EQUAL); + expect(one?.quota).toEqual({ + tokenCount: { totalTokens: 10, inputTokens: 7, cachedInputTokens: 0, outputTokens: 3 }, + modelUsage: [{ model: "fixture-model", tokenCount: { totalTokens: 12, outputTokens: 4 } }], + }); + // Equal counts are not the association: each turn names the user message + // the runtime generated for it, under the same owning record. + expect(typeof one?.promptMessageId).toBe("string"); + expect(typeof two?.promptMessageId).toBe("string"); + expect(one?.promptMessageId).not.toBe(two?.promptMessageId); + expect(one?.acpxRecordId).toBe(handle.acpxRecordId); + expect(two?.acpxRecordId).toBe(handle.acpxRecordId); + expect(typeof one?.sessionId).toBe("string"); + + // And each is exactly the entry the runtime saved for that request. + const saved = world.records.get(handle.acpxRecordId ?? ""); + const perRequest = Reflect.get(saved ?? {}, "request_token_usage") ?? {}; + expect(Object.keys(perRequest).sort()).toEqual( + [one?.promptMessageId, two?.promptMessageId].sort(), + ); + const messages = (Reflect.get(saved ?? {}, "messages") ?? []) as Record[]; + const userIds = messages.flatMap((message) => { + const user = Reflect.get(message, "User"); + return typeof user === "object" && user !== null ? [Reflect.get(user, "id")] : []; + }); + expect(userIds).toEqual([one?.promptMessageId, two?.promptMessageId]); + }); + + it("RU2: an absent response usage stays absent, and a source null stays null", function* () { + const world = yield* useWorld(); + const runtime = runtimeFor(world, [ + {}, + { usage: null, meta: { quota: { token_count: null } } }, + ]); + const handle = yield* ensureHandle(world, runtime, "ru-absent"); + const absent = yield* turnOf(runtime, handle, "alpha"); + const nulled = yield* turnOf(runtime, handle, "beta"); + yield* until(runtime.close({ handle, reason: "test" })); + expect(absent.result.usage !== undefined && "response" in absent.result.usage).toBe(false); + expect(typeof absent.result.usage?.promptMessageId).toBe("string"); + expect(nulled.result.usage?.response).toBe(null); + expect(nulled.result.usage?.quota).toEqual({ tokenCount: null }); + }); + + it("RU3: a response recovered during timeout drain keeps its own usage", function* () { + const world = yield* useWorld(); + const runtime = runtimeFor(world, [{ usage: EQUAL, delayMs: 400 }]); + const handle = yield* ensureHandle(world, runtime, "ru-drain"); + const drained = yield* turnOf(runtime, handle, "alpha", 100); + yield* until(runtime.close({ handle, reason: "test" })); + expect(drained.result.status).toBe("completed"); + expect(drained.result.usage?.response).toEqual(EQUAL); + }); + + it("RU4: a failed finalization keeps the readings already observed", function* () { + const world = yield* useWorld(); + const runtime = runtimeFor(world, [{ usage: EQUAL, meta: { quota: QUOTA } }]); + const handle = yield* ensureHandle(world, runtime, "ru-finalize"); + world.failUsageSaves = true; + const failed = yield* turnOf(runtime, handle, "alpha"); + world.failUsageSaves = false; + yield* until(runtime.close({ handle, reason: "test" }).catch(() => undefined)); + expect(failed.result.status).toBe("failed"); + expect(failed.result.usage?.response).toEqual(EQUAL); + expect(failed.result.usage?.quota?.tokenCount?.totalTokens).toBe(10); + expect(typeof failed.result.usage?.promptMessageId).toBe("string"); + }); + + it("RU5: a turn no backend accepted still carries the readings its result reported", function* () { + const world = yield* useWorld(); + const runtime = runtimeFor(world, [{ usage: EQUAL, accept: false }]); + const handle = yield* ensureHandle(world, runtime, "ru-unaccepted", "first-turn-acceptance"); + const unaccepted = yield* turnOf(runtime, handle, "alpha"); + yield* until(runtime.close({ handle, reason: "test" }).catch(() => undefined)); + expect(unaccepted.materialized).toBe(false); + expect(unaccepted.result.usage?.response).toEqual(EQUAL); + }); + }, +); diff --git a/packages/acp/tests/provider.test.ts b/packages/acp/tests/provider.test.ts index 0e31a8d7c..520b0dfd2 100644 --- a/packages/acp/tests/provider.test.ts +++ b/packages/acp/tests/provider.test.ts @@ -20,6 +20,8 @@ import type { PromptOptions, Session, SessionConfiguration, + PromptUsage, + UsageBuild, } from "@executablemd/core"; import { AcpxSessionPolicyError, @@ -35,7 +37,7 @@ import type { AcpxSessionPolicy, } from "../src/provider.ts"; import { useSerialQueues } from "../src/serial-queue.ts"; -import type { AcpxProvider } from "../src/provider.ts"; +import type { AcpxProvider, ProbeCapableRuntime } from "../src/provider.ts"; import { deriveSessionKey } from "../src/session-key.ts"; import { choice, @@ -49,7 +51,11 @@ import { useFlatWorld, useGitWorld, } from "./helpers.ts"; -import type { AcpPermissionRequest, AcpRuntimeTurnResult } from "../src/acpx-runtime.ts"; +import type { + AcpPermissionRequest, + AcpRuntimeEvent, + AcpRuntimeTurnResult, +} from "../src/acpx-runtime.ts"; import type { CoordinatorHarness, FakeRuntimeHarness } from "./helpers.ts"; import type { NativeAdapter } from "../src/native-launch.ts"; @@ -2960,3 +2966,426 @@ describe("Tier SE — reconfiguring an owned session", () => { }); }); }); + +/** What one observed turn's receiver was handed, in order. */ +interface UsageObservation { + readonly reports: PromptUsage[]; + /** How many receivers the provider asked for. */ + acquisitions: number; +} + +interface ObservedOptions { + /** Offer a receiver at all. */ + observe?: boolean; + /** Raise from the acquisition itself. */ + acquisitionFails?: boolean; + /** Raise from every report. */ + reportFails?: boolean; + /** The carried adapter release, for the semantics table. */ + adapter?: UsageBuild; + /** Construct through the first-turn acceptance barrier under a session policy. */ + strong?: boolean; + /** Wrap the runtime this provider creates. */ + wrap?: (runtime: ProbeCapableRuntime) => ProbeCapableRuntime; +} + +const USAGE_HOST = { host: "deno-source", xmd: { version: "0.0.0-test" } } as const; + +function* installObservedProvider( + harness: FakeRuntimeHarness, + observation: UsageObservation, + options: ObservedOptions = {}, +): Operation { + yield* useFlatWorld(options.strong === true ? WORKSPACE : CWD); + const coordinator = stubCoordinator(); + if (options.observe !== false) { + // deno-lint-ignore require-yield + coordinator.promptUsage = function* () { + observation.acquisitions++; + if (options.acquisitionFails === true) { + throw new Error("acquisition failed"); + } + return (usage: PromptUsage) => { + if (options.reportFails === true) { + throw new Error("receiver failed"); + } + observation.reports.push(usage); + }; + }; + } + const factory = createAcpxProvider({ + createRuntime: + options.wrap === undefined + ? harness.create + : (runtimeOptions) => options.wrap!(harness.create(runtimeOptions)), + sessionStore: makeStore(), + agentRegistry: makeRegistry({ scribe: "scribe-cmd", codex: "codex-cmd", claude: "claude-cmd" }), + advertiseNativeLaunch: [], + advertiseClientNativeAttachment: [], + promptUsageHost: USAGE_HOST, + ...(options.adapter === undefined ? {} : { adapterBuild: () => options.adapter }), + ...(options.strong === true + ? { + nativeAdapters: { + scribe: { + launcher: "scribe", + identity: "provider-returned", + acceptance: "first-turn-acceptance", + resume: () => [], + }, + }, + // deno-lint-ignore require-yield + agentCwd: function* () { + return HOST_DIR; + }, + mcpServers: [], + newSessionOptions: { allowedTools: [], systemPrompt: INSTRUCTIONS }, + permissions: "strict", + sessions: { + // deno-lint-ignore require-yield + *requestSession() { + return { sessionKey: WORKFLOW_SESSION_KEY, cwd: SESSION_DIR, state: "new" }; + }, + *recordSessionIdentity() {}, + }, + } + : {}), + }); + yield* factory({ defaultAgent: "scribe", permissionMode: "deny-all" }, coordinator); +} + +function observation(): UsageObservation { + return { reports: [], acquisitions: 0 }; +} + +/** One turn whose result retained identities and selected readings. */ +function usageResult( + status: "completed" | "cancelled" | "failed", + usage: AcpRuntimeTurnResult["usage"], +): AcpRuntimeTurnResult { + if (status === "failed") { + return { status, error: { message: "turn failed" }, usage }; + } + return { status, stopReason: status === "completed" ? "end_turn" : "cancelled", usage }; +} + +const RETAINED = { + promptMessageId: "message-1", + acpxRecordId: "record-1", + sessionId: "acp-session-1", + response: { inputTokens: 7, outputTokens: 0, totalTokens: 7 }, + quota: { + tokenCount: { totalTokens: 7, inputTokens: 7, reasoningOutputTokens: 0 }, + modelUsage: [{ model: "model-x", tokenCount: { totalTokens: 9 } }], + }, +}; + +const LIVE: AcpRuntimeEvent[] = [ + { type: "text_delta", text: "hello ", stream: "output" }, + { + type: "status", + text: "usage", + tag: "usage_update", + used: 100, + size: 1000, + cost: { amount: 0.25, currency: "USD" }, + }, + { type: "text_delta", text: "world", stream: "output" }, + // Occupancy alone: the latest context replaces the first, and the cost + // already reported stays. + { type: "status", text: "usage", tag: "usage_update", used: 150, size: 1000 }, +]; + +describe("Tier AP — Prompt usage observation", () => { + it("UO1: observing a turn changes none of its events, text or runtime work", function* () { + const runs: { events: AgentPromptEvent[]; close: string; turns: number; ensures: number }[] = + []; + for (const observe of [false, true]) { + const harness = createFakeRuntime(); + harness.script({ events: LIVE, result: usageResult("completed", RETAINED) }); + const seen = observation(); + yield* scoped(function* () { + yield* installObservedProvider(harness, seen, { observe }); + const { events, close } = yield* collectPrompt("hello"); + runs.push({ + events, + close, + turns: harness.turns.length, + ensures: harness.ensureCalls.length, + }); + }); + expect(seen.reports.length).toBe(observe ? 1 : 0); + expect(seen.acquisitions).toBe(observe ? 1 : 0); + } + const strip = (events: AgentPromptEvent[]) => + events.map((event) => (event.type === "started" ? event.type : event)); + expect(strip(runs[1].events)).toEqual(strip(runs[0].events)); + expect(runs[1].close).toBe(runs[0].close); + expect(runs[1].turns).toBe(runs[0].turns); + expect(runs[1].ensures).toBe(runs[0].ensures); + }); + + it("UO2: the report carries the turn's own identities, separate views and replaced live snapshots", function* () { + const harness = createFakeRuntime(); + harness.script({ events: LIVE, result: usageResult("completed", RETAINED) }); + const seen = observation(); + yield* scoped(function* () { + yield* installObservedProvider(harness, seen); + yield* collectPrompt("hello"); + }); + const [usage] = seen.reports; + expect(usage.requestId).toBe(harness.turns[0].turn.requestId); + expect(usage.providerMessageId).toBe("message-1"); + expect(usage.providerRecordId).toBe("record-1"); + expect(usage.providerSessionId).toBe("acp-session-1"); + expect(usage.provenance).toEqual({ + host: "deno-source", + xmd: { version: "0.0.0-test" }, + runtime: { version: "0.21.1" }, + adapter: {}, + }); + expect(usage.readings).toEqual([ + { + source: "response", + scope: "unknown", + kind: "tokens", + counter: "unknown", + tokens: { inputTokens: 7, outputTokens: 0, totalTokens: 7 }, + }, + { + source: "quota-total", + scope: "unknown", + kind: "tokens", + counter: "unknown", + tokens: { totalTokens: 7, inputTokens: 7, reasoningOutputTokens: 0 }, + }, + { + source: "quota-model", + scope: "unknown", + model: "model-x", + kind: "tokens", + counter: "unknown", + tokens: { totalTokens: 9 }, + }, + { source: "update", scope: "session", kind: "context", used: 150, size: 1000 }, + { source: "update", scope: "unknown", kind: "cost", amount: 0.25, currency: "USD" }, + ]); + expect(usage.coverage).toEqual({ + finalReport: "received", + outerPrompt: "unknown", + limitations: [ + "source-semantics-unknown", + "cost-scope-unverified", + "adapter-build-unknown", + "native-build-unobserved", + ], + }); + }); + + it("UO3: a traced adapter release names its own scope, counter and limitations", function* () { + const cases = [ + { + agent: "claude", + adapter: { version: "0.70.0", digest: "sha256:claude" }, + response: { scope: "main-loop", counter: "turn-total" }, + model: { scope: "broader-model-work", counter: "turn-total" }, + limitations: [ + "claude-imputed-zero", + "claude-quota-model-overlaps-response", + "claude-subagent-inclusion-unverified", + ], + }, + { + agent: "codex", + adapter: { version: "1.6.2", digest: "sha256:codex" }, + response: { scope: "last-request", counter: "snapshot" }, + model: { scope: "last-request", counter: "snapshot" }, + limitations: ["codex-last-request-only", "codex-reasoning-inclusion-unverified"], + }, + { + agent: "codex", + adapter: { version: "1.7.0" }, + response: { scope: "unknown", counter: "unknown" }, + model: { scope: "unknown", counter: "unknown" }, + limitations: ["source-semantics-unknown"], + }, + ] as const; + for (const row of cases) { + const harness = createFakeRuntime(); + harness.script({ events: [], result: usageResult("completed", RETAINED) }); + const seen = observation(); + yield* scoped(function* () { + yield* installObservedProvider(harness, seen, { adapter: row.adapter }); + yield* collectPrompt("hello", { agent: row.agent }); + }); + const [usage] = seen.reports; + const [response, , model] = usage.readings; + expect({ + scope: response.scope, + counter: response.kind === "tokens" ? response.counter : "", + }).toEqual(row.response); + expect({ scope: model.scope, counter: model.kind === "tokens" ? model.counter : "" }).toEqual( + row.model, + ); + expect(usage.provenance.adapter).toEqual(row.adapter); + expect(usage.coverage.outerPrompt).toBe( + row.limitations[0] === "source-semantics-unknown" ? "unknown" : "partial", + ); + expect(usage.coverage.limitations).toEqual([...row.limitations, "native-build-unobserved"]); + } + }); + + it("UO4: optional acquisition, report and normalization failures leave the turn as it was", function* () { + const baseline = createFakeRuntime(); + baseline.script({ result: usageResult("completed", RETAINED) }); + const expected = yield* scoped(function* () { + yield* installObservedProvider(baseline, observation(), { observe: false }); + return yield* collectPrompt("hello"); + }); + const phases: { name: string; options: ObservedOptions; result: AcpRuntimeTurnResult }[] = [ + { + name: "acquisition", + options: { acquisitionFails: true }, + result: usageResult("completed", RETAINED), + }, + { + name: "report", + options: { reportFails: true }, + result: usageResult("completed", RETAINED), + }, + { + name: "normalization", + options: {}, + // A selected reading that is not an object at all: normalizing it + // throws inside the collector, which must keep that to itself. + result: usageResult("completed", { ...RETAINED, response: "malformed" as never }), + }, + ]; + for (const phase of phases) { + const harness = createFakeRuntime(); + harness.script({ result: phase.result }); + const seen = observation(); + const observed = yield* scoped(function* () { + yield* installObservedProvider(harness, seen, phase.options); + return yield* collectPrompt("hello"); + }); + const strip = (events: AgentPromptEvent[]) => + events.map((event) => (event.type === "started" ? event.type : event)); + expect({ phase: phase.name, events: strip(observed.events), close: observed.close }).toEqual({ + phase: phase.name, + events: strip(expected.events), + close: expected.close, + }); + expect(harness.turns.length).toBe(1); + expect(harness.turns[0].cancelled).toBe(false); + if (phase.name === "normalization") { + expect(seen.reports.map((usage) => usage.coverage)).toEqual([ + { + finalReport: "unavailable", + outerPrompt: "unknown", + limitations: ["normalization-failed"], + }, + ]); + } else { + expect(seen.reports).toEqual([]); + } + } + }); + + it("UO5: failed and cancelled terminals keep the readings reported before they ended", function* () { + for (const status of ["failed", "cancelled"] as const) { + const harness = createFakeRuntime(); + harness.script({ + events: LIVE, + result: usageResult(status, { promptMessageId: "message-1", acpxRecordId: "record-1" }), + }); + const seen = observation(); + const { events } = yield* scoped(function* () { + yield* installObservedProvider(harness, seen); + return yield* collectPrompt("hello"); + }); + expect(events.find((event) => event.type === "terminal")?.status).toBe(status); + const [usage] = seen.reports; + expect(usage.providerMessageId).toBe("message-1"); + expect(usage.readings.map((reading) => reading.kind)).toEqual(["context", "cost"]); + // The live readings stay; the remainder the turn never reported is + // unknown, not zero. + expect(usage.coverage.finalReport).toBe("unavailable"); + expect(usage.coverage.outerPrompt).toBe("unknown"); + expect(usage.coverage.limitations).toContain("final-report-unavailable"); + } + }); + + it("UO6: a real cleanup failure stays the turn's outcome; usage rides beside it", function* () { + const harness = createFakeRuntime(); + harness.script({ result: usageResult("completed", RETAINED) }); + const seen = observation(); + const { events } = yield* scoped(function* () { + yield* installObservedProvider(harness, seen, { + wrap: (runtime) => ({ + ...runtime, + startTurn(input) { + const turn = runtime.startTurn(input); + let reads = 0; + // The first read of the result fails in transport; the cleanup + // that follows it fails to cancel, then reads the result itself. + return { + ...turn, + get result() { + reads++; + return reads === 1 + ? Promise.reject(new Error("result transport broke")) + : turn.result; + }, + cancel: () => Promise.reject(new Error("cancel failed")), + }; + }, + }), + }); + return yield* collectPrompt("hello"); + }); + const terminal = events.find((event) => event.type === "terminal"); + expect(terminal?.status).toBe("failed"); + expect(terminal?.error).toBeInstanceOf(AggregateError); + expect( + terminal?.error instanceof AggregateError + ? terminal.error.errors.map((error) => error.message) + : [], + ).toEqual(["result transport broke", "cancel failed"]); + expect(seen.reports.map((usage) => usage.providerMessageId)).toEqual(["message-1"]); + }); + + it("UO7: a turn halted before it ended reports nothing and invents no ending", function* () { + const harness = createFakeRuntime(); + harness.script({ manual: true }); + const seen = observation(); + yield* scoped(function* () { + yield* installObservedProvider(harness, seen); + const task = yield* spawn(() => collectPrompt("hello")); + yield* harness.startedTurns(1); + yield* task.halt(); + }); + expect(seen.acquisitions).toBe(1); + expect(seen.reports).toEqual([]); + expect(harness.turns[0].cancelled).toBe(true); + }); + + it("UO8: a turn no backend accepted reports its result before the refusal", function* () { + const harness = createFakeRuntime(); + harness.script({ accepted: false, result: usageResult("completed", RETAINED) }); + const seen = observation(); + let raised: unknown; + yield* scoped(function* () { + yield* installObservedProvider(harness, seen, { strong: true }); + const session = yield* Agent.operations.session(); + try { + yield* collectPrompt("first", { session }); + } catch (error) { + raised = error; + } + }); + expect(raised).toBeInstanceOf(Error); + expect(seen.reports.map((usage) => usage.providerMessageId)).toEqual(["message-1"]); + expect(seen.reports[0].coverage.finalReport).toBe("received"); + }); +}); diff --git a/packages/acp/vendor/acpx/MANIFEST.json b/packages/acp/vendor/acpx/MANIFEST.json index df48db86c..549e6f70b 100644 --- a/packages/acp/vendor/acpx/MANIFEST.json +++ b/packages/acp/vendor/acpx/MANIFEST.json @@ -31,7 +31,7 @@ { "path": "PROVENANCE.md", "kind": "provenance", - "sha256": "c90afafa92a157ff5db4b3b14c1c8042dafe91313365b727278e3f99340ee7c9" + "sha256": "dd0f2c28ee7aabf490a54e7779ac9863fde229f84296a4a051646afe40cf7be6" }, { "path": "generated/agent-registry-Cz9fD9H1.js", @@ -56,17 +56,17 @@ { "path": "generated/queue-owner-runtime-B0mPqOu4.js", "kind": "generated", - "sha256": "39404d45c3a90861cbb29c0d3958c750475a9a5b263b5823470f688e9b62e47d" + "sha256": "4f564876f38b978472f1e1a8f133db53a155f33614b193564a34d5e5b12069bc" }, { "path": "generated/runtime.d.ts", "kind": "generated", - "sha256": "88aedba2daf5c6ecabee5d55f7ac85be1500f2a7f6932d89b1a1688df08a8993" + "sha256": "d7dad6ef20bfbf94867dcd90751ffad58d58bc62cb12469ee5f1524283aa9c87" }, { "path": "generated/runtime.js", "kind": "generated", - "sha256": "30b3da127f6e09470784541b7580ebb79ae52cb0703bb7d1ba1627ab51181e56" + "sha256": "1daede0ebbdf54d2814ce594d695e8b6f1ade183506e09fce799b1bd574b9f4b" }, { "path": "generated/session-options-BlnP3WAc.d.ts", diff --git a/packages/acp/vendor/acpx/PROVENANCE.md b/packages/acp/vendor/acpx/PROVENANCE.md index c5d3324f7..9efbce143 100644 --- a/packages/acp/vendor/acpx/PROVENANCE.md +++ b/packages/acp/vendor/acpx/PROVENANCE.md @@ -137,6 +137,30 @@ and neither origin nor a retryable hint establishes acceptance or permits resend Remove the correction only when a published runtime retains the same selected facts and passes those deterministic consumers and offline vendor verification. +## Selected Prompt usage (#1159) + +The released runtime records a response's `usage` into the session's saved +conversation and returns only its stop reason and `_meta`; a failed or +finalization-failed result keeps neither. Its turn exposes the host request ID, +but not the user-message ID `recordPromptSubmission` generated for that turn. + +The generated correction, bounded by the uniquely named `XMD prompt usage:` seam, +attaches one `usage` member to every terminal result: the generated +`promptMessageId`, its owning `acpxRecordId`, the ACP `sessionId` the turn was +prompted in, the response's selected token counters and the selected +`_meta.quota` `token_count`/`model_usage` counters. Only finite nonnegative +numbers under their original names cross; `null` stays `null`, and a missing or +malformed value stays absent. Timeout-drain recovery carries the recovered +response's own usage, and a failed finalization keeps the readings already +observed. `runPromptTurn` in `queue-owner-runtime-B0mPqOu4.js` hands the +response's `usage` to the runtime for that selection; `runtime.d.ts` declares +the member. Persisted session state, reducer behavior and the user-message +identity ACPX saves are unchanged. + +Remove the correction only when a published runtime exposes the same per-turn +identities and selected readings on every ending and passes the same +deterministic consumers and offline vendor verification. + ## Released environment seam (#566) The released runtime snapshots `agentProcessEnv`, delivers it to every child and diff --git a/packages/acp/vendor/acpx/generated/queue-owner-runtime-B0mPqOu4.js b/packages/acp/vendor/acpx/generated/queue-owner-runtime-B0mPqOu4.js index 07f04d9e2..50a35e118 100644 --- a/packages/acp/vendor/acpx/generated/queue-owner-runtime-B0mPqOu4.js +++ b/packages/acp/vendor/acpx/generated/queue-owner-runtime-B0mPqOu4.js @@ -6709,7 +6709,9 @@ function recoveredSessionResult(response, conversation, promptMessageId) { return { stopReason: response.stopReason, source: "session", - ...responseMetaField(response._meta) + ...responseMetaField(response._meta), + // XMD prompt usage: the recovered response's own usage, for the runtime to select. + ...response.usage === void 0 ? {} : { usage: response.usage } }; } function abortTimedOutRequests(error, lifetime) { @@ -6739,7 +6741,9 @@ async function runPromptTurn(params) { return { stopReason: response.stopReason, source: "rpc", - ...responseMetaField(response._meta) + ...responseMetaField(response._meta), + // XMD prompt usage: the response's own usage, for the runtime to select. + ...response.usage === void 0 ? {} : { usage: response.usage } }; } catch (error) { abortTimedOutRequests(error, lifetime); diff --git a/packages/acp/vendor/acpx/generated/runtime.d.ts b/packages/acp/vendor/acpx/generated/runtime.d.ts index b7f512d03..9b1564b05 100644 --- a/packages/acp/vendor/acpx/generated/runtime.d.ts +++ b/packages/acp/vendor/acpx/generated/runtime.d.ts @@ -310,7 +310,33 @@ type AcpRuntimeTurnResultError = { detailCode?: string; retryable?: boolean; }; -type AcpRuntimeTurnResult = { +/** XMD prompt usage: quota counters, under the adapter's own names. */ +type AcpRuntimeQuotaTokens = { + totalTokens?: number; + inputTokens?: number; + cachedInputTokens?: number; + cachedWriteTokens?: number; + outputTokens?: number; + reasoningOutputTokens?: number; +}; +/** + * XMD prompt usage: what one turn retained about itself, whichever way it ended. + * `promptMessageId` is the user message this turn persisted, under the record + * `acpxRecordId`; `sessionId` is the ACP session it was prompted in. `null` + * is a source that reported null; an absent member is unknown. + */ +type AcpRuntimeTurnUsage = { + promptMessageId?: string; + acpxRecordId?: string; + sessionId?: string; + response?: AcpRuntimeUsageBreakdown | null; + quota?: { + tokenCount?: AcpRuntimeQuotaTokens | null; + modelUsage?: Array<{ model: string; tokenCount: AcpRuntimeQuotaTokens | null }>; + }; +}; +// XMD prompt usage: carried on every terminal result. +type AcpRuntimeTurnResult = ({ status: "completed"; stopReason?: string; _meta?: Record | null; @@ -321,7 +347,8 @@ type AcpRuntimeTurnResult = { } | { status: "failed"; error: AcpRuntimeTurnResultError; -}; +}) & { usage?: AcpRuntimeTurnUsage }; +// XMD prompt usage: the turn result's usage member, exported below. type AcpRuntimeMaterialization = { acpxRecordId: string; agentSessionId?: string }; interface AcpRuntimeTurn { readonly materialized: Promise; @@ -866,4 +893,5 @@ export declare function createAcpRuntime(options: AcpRuntimeOptions): AcpxRuntim export declare function createRuntimeStore(options: { stateDir: string; }): AcpSessionStore; -export { type AcpAgentInspection, type AcpAgentInspectionOptions, type AcpAgentRegistry, type AcpElicitationContext, type AcpElicitationHandler, type AcpElicitationMode, type AcpElicitationRequest, type AcpElicitationResponse, type AcpFileSessionStoreOptions, type AcpInspectableAgentRegistry, type AcpPermissionDecision, type AcpPermissionHandler, type AcpPermissionRequest, type AcpProcessExit, type AcpProcessLaunch, type AcpProcessLaunchScope, type AcpProcessLifecycle, type AcpProcessSpawnFailure, type AcpProcessStarted, type AcpRuntime, type AcpRuntimeMaterialization, type AcpRuntimeAvailableCommand, type AcpRuntimeCapabilities, type AcpRuntimeDoctorReport, type AcpRuntimeEnsureInput, type AcpRuntimeErrorCode, type AcpRuntimeEvent, type AcpRuntimeHandle, type AcpRuntimeOptions, type AcpRuntimePlanEntry, type AcpRuntimePromptMode, type AcpRuntimeSessionContext, type AcpRuntimeSessionMode, type AcpRuntimeSessionModels, type AcpRuntimeSessionPermissions, type AcpRuntimeSessionUsage, type AcpRuntimeStatus, type AcpRuntimeTurn, type AcpRuntimeTurnAttachment, type AcpRuntimeTurnInput, type AcpRuntimeTurnResult, type AcpRuntimeTurnResultError, type AcpRuntimeUsageBreakdown, type AcpRuntimeUsageCost, type AcpSessionRecord, type AcpSessionStore, type AcpSessionUpdateTag, type AcpTextDeltaOriginMeta, DEFAULT_AGENT_NAME, type InspectAgentModelsOptions, type PermissionPolicy, type RequestedModelUnsupportedErrorCode, type RequestedModelUnsupportedReason, type SessionAgentOptions, type SessionWatchEvent, type SessionWatchResult, type SharedAcpRuntimeOptions, type SystemPromptOption, createAgentRegistry }; +// XMD prompt usage: AcpRuntimeTurnUsage and AcpRuntimeQuotaTokens are exported with the turn types. +export { type AcpAgentInspection, type AcpAgentInspectionOptions, type AcpAgentRegistry, type AcpElicitationContext, type AcpElicitationHandler, type AcpElicitationMode, type AcpElicitationRequest, type AcpElicitationResponse, type AcpFileSessionStoreOptions, type AcpInspectableAgentRegistry, type AcpPermissionDecision, type AcpPermissionHandler, type AcpPermissionRequest, type AcpProcessExit, type AcpProcessLaunch, type AcpProcessLaunchScope, type AcpProcessLifecycle, type AcpProcessSpawnFailure, type AcpProcessStarted, type AcpRuntime, type AcpRuntimeMaterialization, type AcpRuntimeAvailableCommand, type AcpRuntimeCapabilities, type AcpRuntimeDoctorReport, type AcpRuntimeEnsureInput, type AcpRuntimeErrorCode, type AcpRuntimeEvent, type AcpRuntimeHandle, type AcpRuntimeOptions, type AcpRuntimePlanEntry, type AcpRuntimePromptMode, type AcpRuntimeSessionContext, type AcpRuntimeSessionMode, type AcpRuntimeSessionModels, type AcpRuntimeSessionPermissions, type AcpRuntimeSessionUsage, type AcpRuntimeStatus, type AcpRuntimeTurn, type AcpRuntimeTurnAttachment, type AcpRuntimeTurnInput, type AcpRuntimeTurnResult, type AcpRuntimeTurnResultError, type AcpRuntimeTurnUsage, type AcpRuntimeQuotaTokens, type AcpRuntimeUsageBreakdown, type AcpRuntimeUsageCost, type AcpSessionRecord, type AcpSessionStore, type AcpSessionUpdateTag, type AcpTextDeltaOriginMeta, DEFAULT_AGENT_NAME, type InspectAgentModelsOptions, type PermissionPolicy, type RequestedModelUnsupportedErrorCode, type RequestedModelUnsupportedReason, type SessionAgentOptions, type SessionWatchEvent, type SessionWatchResult, type SharedAcpRuntimeOptions, type SystemPromptOption, createAgentRegistry }; diff --git a/packages/acp/vendor/acpx/generated/runtime.js b/packages/acp/vendor/acpx/generated/runtime.js index 76525fbf6..8739a3bd1 100644 --- a/packages/acp/vendor/acpx/generated/runtime.js +++ b/packages/acp/vendor/acpx/generated/runtime.js @@ -985,6 +985,46 @@ async function createOrLoadRuntimeSession(client, resumeSessionId, cwd) { }; } const SESSION_MATERIALIZATION_CONTRACT = "executablemd.session-materialization/v1"; +// XMD prompt usage: the typed response and quota counters a turn reported, never raw metadata. +const XMD_PROMPT_USAGE_RESPONSE_FIELDS = ["inputTokens", "outputTokens", "cachedReadTokens", "cachedWriteTokens", "thoughtTokens", "totalTokens"]; +const XMD_PROMPT_USAGE_QUOTA_FIELDS = ["totalTokens", "inputTokens", "cachedInputTokens", "cachedWriteTokens", "outputTokens", "reasoningOutputTokens"]; +// XMD prompt usage: null is the source saying null; a missing or malformed value is unknown. +function xmdPromptUsageCounts(value, fields) { + if (value === null) return null; + if (typeof value !== "object" || Array.isArray(value)) return void 0; + const counts = {}; + for (const field of fields) if (typeof value[field] === "number" && Number.isFinite(value[field]) && value[field] >= 0) counts[field] = value[field]; + return counts; +} +// XMD prompt usage: response.usage, and _meta.quota's token_count and model_usage rows. +function xmdPromptUsageReadings(response) { + const readings = {}; + if (response?.usage !== void 0) { + const counts = xmdPromptUsageCounts(response.usage, XMD_PROMPT_USAGE_RESPONSE_FIELDS); + if (counts !== void 0) readings.response = counts; + } + const quota = response?._meta?.quota; + if (quota !== null && typeof quota === "object" && !Array.isArray(quota)) { + const selected = {}; + // XMD prompt usage: quota counters keep their own names. + if (quota.token_count !== void 0) { + const counts = xmdPromptUsageCounts(quota.token_count, XMD_PROMPT_USAGE_QUOTA_FIELDS); + if (counts !== void 0) selected.tokenCount = counts; + } + if (Array.isArray(quota.model_usage)) selected.modelUsage = quota.model_usage.flatMap((row) => { + if (row === null || typeof row !== "object" || typeof row.model !== "string" || !row.model) return []; + const counts = xmdPromptUsageCounts(row.token_count, XMD_PROMPT_USAGE_QUOTA_FIELDS); + // XMD prompt usage: a row without readable counters says nothing about its model. + return counts === void 0 ? [] : [{ model: row.model, tokenCount: counts }]; + }); + if (Object.keys(selected).length > 0) readings.quota = selected; + } + return readings; +} +// XMD prompt usage: attached to the terminal result whichever way the turn ended. +function withXmdPromptUsage(result, usage) { + return Object.keys(usage).length === 0 ? result : { ...result, usage: { ...usage } }; +} function sessionMaterializationPending(record) { return record?.sessionMaterialization?.state === "pending"; } @@ -1747,6 +1787,8 @@ var AcpRuntimeManager = class { async runRuntimeTurnTask(task) { let turn; let terminalResult; + // XMD prompt usage: identities and selected readings this turn retains on every ending. + const usage = {}; try { if (this.cancelRuntimeTurnBeforePrompt(task)) { await this.closeRetainedOneShotHandle(task.input.handle); @@ -1756,7 +1798,12 @@ var AcpRuntimeManager = class { }; } else { turn = await this.prepareRuntimeTurn(task); + // XMD prompt usage: the generated user message and its owning record. + usage.promptMessageId = turn.promptMessageId; + usage.acpxRecordId = turn.record.acpxRecordId; const { sessionId } = await this.connectRuntimeTurn(task, turn); + // XMD prompt usage: the ACP session this turn was loaded into and prompted. + if (typeof sessionId === "string" && sessionId) usage.sessionId = sessionId; this.assertOpen(); await this.resolveRuntimeTurnReady(task, turn); if (this.cancelRuntimeTurnBeforePrompt(task)) terminalResult = { @@ -1765,6 +1812,8 @@ var AcpRuntimeManager = class { }; else { const response = await this.runRuntimePrompt(task, turn, sessionId); + // XMD prompt usage: selected before the response metadata is projected. + Object.assign(usage, xmdPromptUsageReadings(response)); this.updateCompletedRuntimeTurn(turn); terminalResult = { status: response.stopReason === "cancelled" ? "cancelled" : "completed", @@ -1777,11 +1826,12 @@ var AcpRuntimeManager = class { terminalResult = this.handleRuntimeTurnFailure(task, error); } const finalization = await settleAttempt(async () => this.serializeManagerOperation(this.runtimeOperationLocks, task.input.handle.acpxRecordId ?? task.input.handle.sessionKey, async () => this.finalizeRuntimeTurn(task, turn))); + // XMD prompt usage: a failed finalization keeps the readings already observed. if (!finalization.ok) { - task.settleResult(this.failRuntimeTurn(task, finalization.error)); + task.settleResult(withXmdPromptUsage(this.failRuntimeTurn(task, finalization.error), usage)); throw finalization.error; } - task.settleResult(terminalResult); + task.settleResult(withXmdPromptUsage(terminalResult, usage)); } async runRuntimePrompt(task, turn, sessionId) { return await runPromptTurn({ diff --git a/packages/cli/src/agent-stack.ts b/packages/cli/src/agent-stack.ts index bbce0e79e..fe5527e79 100644 --- a/packages/cli/src/agent-stack.ts +++ b/packages/cli/src/agent-stack.ts @@ -151,6 +151,7 @@ export function hostAcpDependencies(stack: PlanWriterStack): AcpxProviderDepende const adapters = { ...embeddedAdapterDependencies(stack.adapters), sessionStore: createAcpxSessionStore(join(homedir(), ".acpx")), + ...(sessions?.usage === undefined ? {} : { promptUsageHost: sessions.usage }), }; if (sessions === undefined) { return adapters; diff --git a/packages/cli/src/compiled.ts b/packages/cli/src/compiled.ts index 7f80886b8..4a457adc0 100644 --- a/packages/cli/src/compiled.ts +++ b/packages/cli/src/compiled.ts @@ -23,6 +23,7 @@ import { denoRunRepositories, selectReplEntryHost } from "./deno-repositories.ts import { BundledRunPlugin } from "./run-profile.ts"; import { isCredentialHelperMode, runCredentialHelper } from "@executablemd/git/credential-helper"; import type { HelperAssembly } from "@executablemd/git/credential-helper"; +import type { PromptUsageHost } from "@executablemd/acp"; import { useCompiledService } from "./compiled-service.ts"; /** @@ -52,6 +53,13 @@ const UPGRADE = compiledUpgradeAssembly({ currentVersion: XMD_VERSION, }); +/** + * What this host states about its own build for the Prompt usage its turns + * retain. The version is this build's own; a digest or source revision this + * process has not measured is left unknown rather than guessed. + */ +const USAGE: PromptUsageHost = { host: "compiled", xmd: { version: XMD_VERSION } }; + // Before anything public is parsed, and absent from every public surface. if (isReplSubmit(process.argv.slice(2))) { try { @@ -141,8 +149,8 @@ if (isReplSubmit(process.argv.slice(2))) { UPGRADE, () => readInputStream(process.stdin), importPluginModule, - () => useDenoWorkflowHost(HELPER), - useMachineSessions(), + () => useDenoWorkflowHost(HELPER, USAGE), + { ...useMachineSessions(), usage: USAGE }, // What this host selects for a REPL command, and what it installs for // one. The selection is the capability an entry runs with — its // checkouts, its local Git, its publications and its issues; Node and Bun diff --git a/packages/cli/src/deno-workflow.ts b/packages/cli/src/deno-workflow.ts index c0421da5f..6908d8182 100644 --- a/packages/cli/src/deno-workflow.ts +++ b/packages/cli/src/deno-workflow.ts @@ -19,6 +19,7 @@ * directory being involved at all. */ +import type { PromptUsageHost } from "@executablemd/acp"; import { homedir } from "node:os"; import { join } from "node:path"; import type { Operation } from "effection"; @@ -42,7 +43,10 @@ export const DEFAULT_RUN_STORAGE_ROOT: string = join(homedir(), ".xmd", "runs"); /** The variable that names a different run store. Absolute, or it is refused. */ export const RUN_STORAGE_ROOT_ENV = "XMD_WORKFLOW_RUNS"; -export function* useDenoWorkflowHost(helper: HelperAssembly): Operation { +export function* useDenoWorkflowHost( + helper: HelperAssembly, + usage?: PromptUsageHost, +): Operation { const configured = yield* readEnv(RUN_STORAGE_ROOT_ENV); const root = configured === undefined || configured === "" ? DEFAULT_RUN_STORAGE_ROOT : configured; @@ -71,7 +75,12 @@ export function* useDenoWorkflowHost(helper: HelperAssembly): Operation useWorkflowAgentProfile({ root, attachment }), + agent: (attachment) => + useWorkflowAgentProfile({ + root, + attachment, + ...(usage === undefined ? {} : { promptUsageHost: usage }), + }), }); }, }; diff --git a/packages/cli/src/deno.ts b/packages/cli/src/deno.ts index c9e421117..8821f1a06 100644 --- a/packages/cli/src/deno.ts +++ b/packages/cli/src/deno.ts @@ -26,6 +26,7 @@ import { denoRunRepositories, selectReplEntryHost } from "./deno-repositories.ts import { BundledRunPlugin } from "./run-profile.ts"; import { isCredentialHelperMode, runCredentialHelper } from "@executablemd/git/credential-helper"; import type { HelperAssembly } from "@executablemd/git/credential-helper"; +import type { PromptUsageHost } from "@executablemd/acp"; import { useDenoService } from "./deno-service.ts"; const ENTRYPOINT = fileURLToPath(import.meta.url); @@ -66,6 +67,13 @@ const UPGRADE: UpgradeAssembly = { architecture: process.arch, }; +/** + * What this host states about its own build for the Prompt usage its turns + * retain. The version is this build's own; a digest or source revision this + * process has not measured is left unknown rather than guessed. + */ +const USAGE: PromptUsageHost = { host: "deno-source", xmd: { version: XMD_VERSION } }; + // The internal helper mode runs before anything public is parsed. It is not a // command: it appears in no help and in no public grammar, and a caller who did // not select it gets the ordinary command line unchanged. @@ -160,8 +168,8 @@ if (isReplSubmit(process.argv.slice(2))) { UPGRADE, () => readInputStream(process.stdin), importPluginModule, - () => useDenoWorkflowHost(HELPER), - useMachineSessions(), + () => useDenoWorkflowHost(HELPER, USAGE), + { ...useMachineSessions(), usage: USAGE }, // What this host selects for a REPL command, and what it installs for // one. The selection is the capability an entry runs with — its // checkouts, its local Git, its publications and its issues; Node and Bun diff --git a/packages/cli/src/session-coordinator.ts b/packages/cli/src/session-coordinator.ts index 6f913e418..bbc5ba78e 100644 --- a/packages/cli/src/session-coordinator.ts +++ b/packages/cli/src/session-coordinator.ts @@ -25,7 +25,7 @@ import { ADVERTISED_NATIVE_LAUNCH, createDenoSessionRouteStore, } from "@executablemd/acp"; -import type { AgentSessionRouteStore } from "@executablemd/acp"; +import type { AgentSessionRouteStore, PromptUsageHost } from "@executablemd/acp"; export function sessionCoordinatorRoot(): string { return join(homedir(), ".acpx", "xmd-native-sessions", "v1"); @@ -48,6 +48,8 @@ export interface MachineSessionAssembly { executableObserver?: ExecutableObserver; advertiseNativeLaunch: readonly string[]; advertiseClientNativeAttachment: readonly string[]; + /** What this host states about its own build, for the Prompt usage its turns retain. */ + usage?: PromptUsageHost; } /** This host's session coordinator, or nothing when it cannot provide one. */ diff --git a/packages/cli/src/workflow-agent.ts b/packages/cli/src/workflow-agent.ts index 3104b9e0a..d78a4321f 100644 --- a/packages/cli/src/workflow-agent.ts +++ b/packages/cli/src/workflow-agent.ts @@ -96,6 +96,7 @@ import { embeddedAdapterDependencies, } from "@executablemd/acp/embedded-adapters"; import type { EmbeddedAdapters } from "@executablemd/acp/embedded-adapters"; +import type { PromptUsageHost } from "@executablemd/acp"; import { installAgentComponents, installPermissionMode, @@ -221,6 +222,8 @@ export interface WorkflowAgentProfileOptions { * without a real install. */ readonly adapters?: EmbeddedAdapters; + /** What this host states about its own build, for the Prompt usage its turns retain. */ + readonly promptUsageHost?: PromptUsageHost; } /** @@ -437,6 +440,7 @@ export function* useWorkflowAgentProfile(options: WorkflowAgentProfileOptions): ...embeddedAdapterDependencies(adapters), ...(options.createRuntime === undefined ? {} : { createRuntime: options.createRuntime }), ...(options.agentProcessEnv === undefined ? {} : { agentProcessEnv: options.agentProcessEnv }), + ...(options.promptUsageHost === undefined ? {} : { promptUsageHost: options.promptUsageHost }), // ACP-only, stated rather than inherited. A workflow session belongs to a // run, not to this machine: it is named by a row in the run's own database, // arranged in the run's own sidecar, and continued by reattaching that row. diff --git a/packages/cli/tests/agent-usage-calibration.test.ts b/packages/cli/tests/agent-usage-calibration.test.ts new file mode 100644 index 000000000..98fd1ffce --- /dev/null +++ b/packages/cli/tests/agent-usage-calibration.test.ts @@ -0,0 +1,197 @@ +/** + * Tier UC — the usage calibration driver (examples/agent-usage/README.md). + * + * The grammar and its refusals, and one complete experiment through the + * production source CLI against the controlled native Claude fixture — no + * credentials and no model call. A controlled agent leaves no native + * transcript, so the experiment must end incomplete, keep what it did measure, + * and keep every private identity, path and marker out of the shareable report. + */ + +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { ensure, scoped } from "effection"; +import type { Operation } from "effection"; +import { ensureDir, exists, readTextFile, rm } from "@effectionx/fs"; +import { exec } from "@executablemd/runtime"; +import { useQuietProcessOutput } from "@executablemd/runtime"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; +import { HELP, parseSelection, Refusal } from "../../../examples/agent-usage/selection.ts"; + +const CHECKOUT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +const DRIVER = join(CHECKOUT, "examples/agent-usage/calibrate.ts"); +const FAKE_CLAUDE = join(CHECKOUT, "packages/acp/tests/fixtures/fake-claude-cli.cjs"); + +function* useRoot(): Operation { + const root = join(tmpdir(), `xmd-uc-${randomUUID()}`); + yield* ensureDir(root); + yield* ensure(() => rm(root, { recursive: true, force: true })); + return root; +} + +/** The parent's environment, without the members it does not set. */ +function inherited(): Record { + return Object.fromEntries( + Object.entries(process.env).flatMap(([name, value]) => + value === undefined ? [] : [[name, value]], + ), + ); +} + +function* driver( + args: string[], + env: Record = {}, +): Operation<{ code: number; stdout: string; stderr: string }> { + return yield* scoped(function* () { + yield* useQuietProcessOutput(); + const result = yield* exec({ + command: [process.execPath, "run", "--allow-all", "--frozen", DRIVER, ...args], + env: { ...inherited(), ...env }, + retain: true, + timeout: 300_000, + }); + return { code: result.exitCode, stdout: result.stdout, stderr: result.stderr }; + }); +} + +describe("Tier UC — usage calibration driver", () => { + it("UC1: the grammar selects exactly one experiment and refuses everything else first", function* () { + const valid = ["--host=source", "--agent=claude", "--scenario=success", "--output=/o"]; + expect(parseSelection(["--help", "--bogus"])).toEqual({ kind: "help" }); + expect(parseSelection([...valid, "--allow-paid"])).toEqual({ + kind: "selected", + selection: { host: "source", agent: "claude", scenario: "success", output: "/o" }, + }); + expect(parseSelection(valid)).toEqual({ kind: "refused", message: Refusal.paid }); + const unsupported = { kind: "refused", message: Refusal.unsupported }; + for (const args of [ + [...valid, "--allow-paid", "--scenario=resume"], + [...valid, "--allow-paid", "--allow-paid"], + [...valid, "--allow-paid", "positional"], + [...valid, "--allow-paid", "--colour=red"], + ["--host=source", "--agent=gemini", "--scenario=success", "--output=/o", "--allow-paid"], + ["--host=source", "--agent=claude", "--scenario=everything", "--output=/o", "--allow-paid"], + ["--host=source", "--agent=claude", "--scenario=success", "--allow-paid"], + ["--host=source", "--agent=claude", "--scenario=success", "--output=", "--allow-paid"], + [...valid, "--xmd=/bin/xmd", "--allow-paid"], + ]) { + expect(parseSelection(args)).toEqual(unsupported); + } + const compiled = [ + "--host=compiled", + "--agent=codex", + "--scenario=cancel", + "--output=/o", + "--allow-paid", + ]; + expect(parseSelection(compiled)).toEqual({ kind: "refused", message: Refusal.compiled }); + expect(parseSelection([...compiled, "--xmd=relative/xmd"])).toEqual({ + kind: "refused", + message: Refusal.compiled, + }); + expect(parseSelection([...compiled, "--xmd=/abs/xmd"])).toEqual({ + kind: "selected", + selection: { + host: "compiled", + agent: "codex", + scenario: "cancel", + output: "/o", + xmd: "/abs/xmd", + }, + }); + }); + + it("UC2: help and refusals print their exact text and create nothing", function* () { + const root = yield* useRoot(); + const help = yield* driver(["--help"]); + expect([help.code, help.stdout.trim()]).toEqual([0, HELP.trim()]); + const output = join(root, "never"); + const refused = yield* driver([ + "--host=source", + "--agent=claude", + "--scenario=success", + `--output=${output}`, + ]); + expect([refused.code, refused.stderr.trim(), refused.stdout]).toEqual([1, Refusal.paid, ""]); + expect(yield* exists(output)).toBe(false); + yield* ensureDir(join(root, "taken")); + const taken = yield* driver([ + "--host=source", + "--agent=claude", + "--scenario=success", + `--output=${join(root, "taken")}`, + "--allow-paid", + ]); + expect([taken.code, taken.stderr.trim()]).toEqual([1, Refusal.exists(join(root, "taken"))]); + }); + + it("UC3: a controlled experiment keeps what it measured, ends incomplete and shares no private value", function* () { + const root = yield* useRoot(); + const output = join(root, "claude-source-success"); + const ran = yield* driver( + [ + "--host=source", + "--agent=claude", + "--scenario=success", + `--output=${output}`, + "--allow-paid", + ], + { CLAUDE_CODE_EXECUTABLE: FAKE_CLAUDE }, + ); + expect([ran.code, ran.stdout.trim()]).toEqual([ + 1, + `Calibration incomplete. Read ${output}/report.md before attempting another run.`, + ]); + const report = JSON.parse(yield* readTextFile(join(output, "report.json"))); + expect(report.result).toBe("incomplete"); + expect([report.allowance, report.submittedPrompts, report.acknowledgedPrompts]).toEqual([ + 2, 2, 2, + ]); + expect(report.scenarioChecks.twoPromptsOneSession.outcome).toBe("verified"); + // A controlled agent writes no native transcript: the required multicall + // witness is missing, which is incomplete calibration, not a pass. + expect(report.scenarioChecks.multicall).toEqual({ + outcome: "unavailable", + reason: "native-tool-evidence-unavailable", + }); + expect(report.teardown.outcome).toBe("verified"); + for (const prompt of report.prompts) { + expect(prompt.checks.usage.outcome).toBe("verified"); + expect(prompt.checks.association).toEqual({ + outcome: "verified", + reason: "generated-message-in-owning-record", + }); + expect(prompt.checks.persistedAgreement.outcome).toBe("verified"); + } + expect(report.provenance.adapter.version).toBe("0.70.0"); + const support = JSON.parse(yield* readTextFile(join(output, "support/association.json"))); + const privateValues: string[] = [output, root]; + for (const prompt of support.prompts) { + privateValues.push( + prompt.usage.providerMessageId, + prompt.usage.providerRecordId, + prompt.sessionKey, + ); + } + for (const file of ["report.json", "report.md"]) { + const shared = yield* readTextFile(join(output, file)); + for (const value of privateValues) { + expect({ file, leaked: shared.includes(value) }).toEqual({ file, leaked: false }); + } + expect(shared.includes("USAGE-A-")).toBe(false); + } + const index = JSON.parse(yield* readTextFile(join(output, "evidence-index.json"))); + expect(index.files.map((file: { path: string }) => file.path)).toEqual( + expect.arrayContaining([ + "report.json", + "report.md", + "success.jsonl", + "support/association.json", + ]), + ); + }); +}); diff --git a/packages/cli/tests/prompt-usage-hosts.test.ts b/packages/cli/tests/prompt-usage-hosts.test.ts new file mode 100644 index 000000000..bf1736355 --- /dev/null +++ b/packages/cli/tests/prompt-usage-hosts.test.ts @@ -0,0 +1,245 @@ +/** + * Tier HU — Prompt usage through each concrete host assembly + * (specs/acp-client-spec.md §Prompt usage). + * + * Every row drives the product's own assembly — the agent stack, the Plan + * writer's dependencies, the REPL profile and its headless wrapper, the + * workflow Agent profile and the Sidekick's installation — and replaces only + * the runtime leaf. What each row asserts is the canonical record the history + * actually retains and reads back, not that a dependency was constructed. + */ + +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import type { DurableEvent, DurableStream } from "@executablemd/durable-streams"; +import { scoped, useScope } from "effection"; +import type { Operation } from "effection"; +import { ensureDir, rm } from "@effectionx/fs"; +import { ensure } from "effection"; +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { useHostFiles } from "@executablemd/runtime"; +import { agentIdentityComponents, collect, retainedSource } from "@executablemd/core"; +import type { PromptUsage } from "@executablemd/core"; +import { executeInstalled, parsePromptRecord, useAgentConversation } from "@executablemd/core/host"; +import { withWorkflowWorkspace } from "@executablemd/workflow/deno"; +import type { AcpxProviderDependencies, PromptUsageHost } from "@executablemd/acp"; +import { createEmbeddedAdapters } from "@executablemd/acp/embedded-adapters"; +import type { AgentStack } from "../src/agent-stack.ts"; +import { hostAcpDependencies, installAgentProviderStack } from "../src/agent-stack.ts"; +import { planWriterDependencies } from "../src/plan-writer-profile.ts"; +import { assembleReplProfile } from "../src/repl-profile.ts"; +import { NO_PLUGINS } from "../src/plugin-host.ts"; +import { sidekickAssembly } from "../src/repl/sidekick/assembly.ts"; +import { useWorkflowAgentProfile } from "../src/workflow-agent.ts"; +import { createFakeAcp, makeStore } from "./support/fake-acp.ts"; +import type { FakeAcp } from "./support/fake-acp.ts"; +import { createRun, useStorageRoot, withStorage } from "./support/workflow-run.ts"; + +const HOST: PromptUsageHost = { host: "deno-source", xmd: { version: "0.0.0-hosts" } }; + +const RETAINED = { + promptMessageId: "message-hosts", + acpxRecordId: "record-hosts", + sessionId: "acp-session-hosts", + response: { inputTokens: 11, outputTokens: 2, totalTokens: 13 }, +}; + +function* useRoot(): Operation { + const root = join(tmpdir(), `xmd-hu-${randomUUID()}`); + yield* ensureDir(root); + yield* ensure(() => rm(root, { recursive: true, force: true })); + return root; +} + +function stackAt(root: string, defaultAgent = "claude"): AgentStack { + const embedded = createEmbeddedAdapters(join(root, "adapters")); + return { + provider: "acpx", + defaultAgent, + permissionMode: "deny-all", + adapters: { ...embedded, *materialize() {} }, + sessions: { advertiseNativeLaunch: [], advertiseClientNativeAttachment: [], usage: HOST }, + }; +} + +function leaf(stack: AgentStack, fake: FakeAcp): AcpxProviderDependencies { + return { + ...hostAcpDependencies(stack), + createRuntime: fake.create, + sessionStore: makeStore(), + }; +} + +function usageIn(events: readonly DurableEvent[]): (PromptUsage | undefined)[] { + return events.flatMap((event) => + event.type === "yield" && + event.description.type === "agent_prompt" && + event.result.status === "ok" + ? [parsePromptRecord(event.result.value)?.usage] + : [], + ); +} + +function* promptOnce(stream: DurableStream): Operation { + yield* collect( + yield* executeInstalled( + { ...retainedSource("hosts.md", "measure me"), stream }, + [{ components: agentIdentityComponents() }], + ), + ); +} + +function expectRetained(usage: PromptUsage | undefined, adapterVersion: string): void { + expect(usage?.providerMessageId).toBe("message-hosts"); + expect(usage?.providerRecordId).toBe("record-hosts"); + expect(usage?.provenance.host).toBe("deno-source"); + expect(usage?.provenance.xmd).toEqual({ version: "0.0.0-hosts" }); + expect(usage?.provenance.adapter.version).toBe(adapterVersion); + expect(usage?.readings[0]).toEqual({ + source: "response", + scope: adapterVersion === "0.70.0" ? "main-loop" : "last-request", + kind: "tokens", + counter: adapterVersion === "0.70.0" ? "turn-total" : "snapshot", + tokens: { inputTokens: 11, outputTokens: 2, totalTokens: 13 }, + }); +} + +describe("Tier HU — Prompt usage through the host assemblies", () => { + it("HU1: an ordinary run's agent stack retains usage in the run's own history", function* () { + yield* useHostFiles(); + const root = yield* useRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + const stream = new InMemoryStream(); + yield* scoped(function* () { + const stack = stackAt(root); + yield* installAgentProviderStack(stack, leaf(stack, fake)); + yield* promptOnce(stream); + }); + const [usage] = usageIn(yield* stream.readAll()); + expectRetained(usage, "0.70.0"); + }); + + it("HU2: the Plan writer's dependencies retain usage the same way", function* () { + yield* useHostFiles(); + const root = yield* useRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + const stream = new InMemoryStream(); + yield* scoped(function* () { + const stack = stackAt(root); + const host = yield* useScope(); + yield* installAgentProviderStack( + stack, + planWriterDependencies({ stack, acp: leaf(stack, fake) }, root, host), + ); + yield* promptOnce(stream); + }); + expectRetained(usageIn(yield* stream.readAll())[0], "0.70.0"); + }); + + it("HU3: the REPL profile, through its headless wrapper, retains usage", function* () { + yield* useHostFiles(); + const root = yield* useRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + const stream = new InMemoryStream(); + yield* scoped(function* () { + const stack = stackAt(root); + yield* assembleReplProfile(stack, NO_PLUGINS, { + acp: leaf(stack, fake), + planWriterRoot: join(root, "plans"), + }); + yield* promptOnce(stream); + }); + expectRetained(usageIn(yield* stream.readAll())[0], "0.70.0"); + }); + + it("HU4: the Sidekick's held conversation retains usage through its own assembly", function* () { + yield* useHostFiles(); + const root = yield* useRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + const history = new InMemoryStream(); + const stack = stackAt(root); + const assembly = sidekickAssembly("claude", leaf(stack, fake)); + const answered = yield* scoped(function* () { + const opened = yield* useAgentConversation({ + history, + id: "sidekick-usage", + agent: "claude", + installations: assembly.installations, + }); + if (!opened.ok) { + throw opened.error; + } + return yield* opened.value.prompt("measure me"); + }); + expect(answered.ok ? "ok" : answered.error.message).toBe("ok"); + expectRetained(usageIn(yield* history.readAll())[0], "0.70.0"); + }); + + it("HU5: the workflow Agent profile retains usage in the run's journal", function* () { + yield* useHostFiles(); + const root = yield* useStorageRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + yield* withStorage(root, function* () { + const database = yield* createRun(); + yield* withWorkflowWorkspace( + database, + scoped(function* () { + return yield* collect( + yield* executeInstalled( + { + ...retainedSource( + "workflows/usage.md", + 'measure me', + ), + stream: database.journal, + }, + [{ components: [...agentIdentityComponents()] }], + ), + ); + }), + { + agent: (attachment) => + useWorkflowAgentProfile({ + // deno-lint-ignore require-yield + agentProcessEnv: function* () { + return undefined; + }, + root, + attachment, + defaultAgent: "codex", + sessionStore: makeStore(), + createRuntime: fake.create, + adapters: { ...createEmbeddedAdapters(join(root, "adapters")), *materialize() {} }, + promptUsageHost: HOST, + }), + }, + ); + expectRetained(usageIn(yield* database.journal.readAll())[0], "1.6.2"); + }); + }); + + it("HU6: a host that states no usage facts collects none, and runs the same turn", function* () { + yield* useHostFiles(); + const root = yield* useRoot(); + const fake = createFakeAcp(); + fake.script({ reply: "ok", usage: RETAINED }); + const stream = new InMemoryStream(); + yield* scoped(function* () { + const stack: AgentStack = { + ...stackAt(root), + sessions: { advertiseNativeLaunch: [], advertiseClientNativeAttachment: [] }, + }; + yield* installAgentProviderStack(stack, leaf(stack, fake)); + yield* promptOnce(stream); + }); + expect(usageIn(yield* stream.readAll())).toEqual([undefined]); + }); +}); diff --git a/packages/cli/tests/support/fake-acp.ts b/packages/cli/tests/support/fake-acp.ts index c52f18ac3..fd7bc6a7c 100644 --- a/packages/cli/tests/support/fake-acp.ts +++ b/packages/cli/tests/support/fake-acp.ts @@ -187,6 +187,12 @@ export interface ScriptedTurn { * turns answer. */ readonly turnId?: string; + /** + * What the runtime retained about this turn's usage, on whichever result it + * settles with — the identities and selected readings the vendored runtime + * attaches to every terminal result. + */ + readonly usage?: AcpRuntimeTurnResult["usage"]; } /** One value a scripted agent offers for a setting. */ @@ -898,15 +904,17 @@ export function createFakeAcp(): FakeAcp { } else { void allowed.then( () => { + const usage = turn.usage === undefined ? {} : { usage: turn.usage }; if (turn.failure !== undefined) { settled.resolve({ status: "failed", error: { message: turn.failure, ...turn.failureFacts }, + ...usage, }); return; } if (turn.cancelled === true) { - settled.resolve({ status: "cancelled" }); + settled.resolve({ status: "cancelled", ...usage }); return; } settled.resolve({ @@ -915,6 +923,7 @@ export function createFakeAcp(): FakeAcp { ...(turn.turnId === undefined ? {} : { _meta: { codex: { turnId: turn.turnId } } }), + ...usage, }); }, (error) => { diff --git a/packages/core/mod.ts b/packages/core/mod.ts index 5585d07dc..ccc30d6fe 100644 --- a/packages/core/mod.ts +++ b/packages/core/mod.ts @@ -398,6 +398,17 @@ export type { export { AgentLaunchProtocolError } from "./src/agent/launch-request.ts"; export type { AgentLaunchRequest } from "./src/agent/launch-request.ts"; export type { AgentLaunchPhases, AgentLaunchCoordinator } from "./src/agent/launch-coordinator.ts"; +// What a provider reports through the coordinator's optional usage receiver, +// and what a canonical Prompt record retains of it. +export type { + PromptUsage, + PromptUsageReporter, + ReportedTokens, + UsageBuild, + UsageCoverage, + UsageProvenance, + UsageReading, +} from "./src/agent/usage.ts"; // What settles one session setup. A provider receives the handle through the // coordinator delivered to its factory; the types are here so it can name them. export type { diff --git a/packages/core/src/agent/function-components.ts b/packages/core/src/agent/function-components.ts index 178560150..6399f3790 100644 --- a/packages/core/src/agent/function-components.ts +++ b/packages/core/src/agent/function-components.ts @@ -60,9 +60,11 @@ import type { SerializedPromptFailure } from "./errors.ts"; import { auditedPromptStream, completedPromptAudit, + completedPromptUsage, persistPrompt, promptFailureFromRecord, } from "./journal.ts"; +import { retainablePromptUsage } from "./usage.ts"; import type { PromptRecord, RetainPromptInput } from "./journal.ts"; import { checkpointOf } from "./checkpoint.ts"; import type { AgentPromptCheckpoint } from "./checkpoint.ts"; @@ -1039,6 +1041,20 @@ export function* runPrompt( if (answered.length > 0) { record.permissions = answered; } + // Read from the same exact stream, after the same joined teardown, and + // selected here because this is still the execution that produced the turn. + // Usage is observation: if reading or selecting it fails, the record simply + // carries none, and the turn's outcome is exactly what it was. + if (audited !== undefined) { + try { + const reported = yield* completedPromptUsage(audited); + if (reported !== undefined) { + record.usage = yield* retainablePromptUsage(reported); + } + } catch { + // Unknown usage is what an absent member already says. + } + } if (throwOnError && status !== "completed") { record.raised = true; } diff --git a/packages/core/src/agent/journal.ts b/packages/core/src/agent/journal.ts index 32e33683d..7d5eeb0b6 100644 --- a/packages/core/src/agent/journal.ts +++ b/packages/core/src/agent/journal.ts @@ -55,7 +55,7 @@ import type { Result as DurableResult, Workflow, } from "@executablemd/durable-streams"; -import { createContext } from "effection"; +import { createContext, ensure } from "effection"; import type { Operation, Stream } from "effection"; import { Agent } from "./agent-api.ts"; import { InvocationTeardownError } from "../invocation.ts"; @@ -76,6 +76,13 @@ import { AgentInternal } from "./internal.ts"; import { HostObservationFailure } from "./publication.ts"; import type { AgentPromptAssociation, AgentPromptHandle } from "./publication.ts"; import { SourceOwner, sourceOwnerDescription } from "../source-owner.ts"; +import { + canonicalUsage, + promptUsageDestination, + readPromptUsage, + serializePromptUsage, +} from "./usage.ts"; +import type { PromptUsage, PromptUsageDestination, PromptUsageReporter } from "./usage.ts"; import type { SourcePosition } from "../types.ts"; /** The durable effect type every journaled Agent Prompt is recorded under. */ @@ -146,6 +153,15 @@ export interface PromptRecord { * it happened if it names which turn it was. */ checkpoint?: AgentPromptCheckpoint; + /** + * What the provider reported about this turn's consumption, when it reported + * anything this execution could retain. + * + * Absent when nothing was reported, and on every record written before this + * member existed — both mean unknown, never zero. A failed or cancelled turn + * keeps what was reported before it ended; its coverage says what is missing. + */ + usage?: PromptUsage; } /** The safe half of one request, before its decision has been made. */ @@ -191,6 +207,32 @@ export interface PromptPermissionAudit { completed(): readonly PromptPermission[]; } +/** + * Where the current subscription's provider reports usage. + * + * Placed around one actual subscription and its `next()` calls, so a provider + * asking from inside its own stream finds the destination of the turn it is + * running and no other. Private to this module, like the permission ledger. + */ +const PromptUsageAudit = createContext("xmd.agent.prompt-usage-audit"); + +/** + * The receiver for the subscription being started, if one is being audited. + * + * Reached only through the launch coordinator Core delivers to an installed + * provider. Undefined outside an audited subscription, which leaves usage + * unavailable and refuses nothing. + */ +export function* currentPromptUsageReporter(): Operation { + return (yield* PromptUsageAudit.get())?.report; +} + +/** One wrapped stream's audits: its permission ledger, and one usage slot per subscription. */ +interface StreamAudit { + readonly permissions: PromptPermissionAudit; + readonly usage: PromptUsageDestination[]; +} + /** * Observe every permission decision, from outside every policy that makes one. * @@ -246,8 +288,8 @@ export function* observePermissionDecisions(): Operation { export function* auditedPromptStream( stream: Stream, ): Operation> { - const audit = promptPermissionAudit(); - const wrapped = audited(stream, audit.ledger); + const audit: StreamAudit = { permissions: promptPermissionAudit(), usage: [] }; + const wrapped = audited(stream, audit); (yield* PromptAudits.get())?.set(wrapped, audit); return wrapped; } @@ -259,9 +301,7 @@ export function* auditedPromptStream( * can reach the map, and holding a stream is the only way to name an entry. */ const PromptAudits = - createContext, PromptPermissionAudit>>( - "xmd.agent.prompt-audits", - ); + createContext, StreamAudit>>("xmd.agent.prompt-audits"); /** * What the turn consumed through this exact stream was allowed to do. @@ -273,7 +313,23 @@ export function* completedPromptAudit( stream: Stream, ): Operation { const audits = yield* PromptAudits.get(); - return audits?.get(stream)?.completed(); + return audits?.get(stream)?.permissions.completed(); +} + +/** + * What the turn consumed through this exact stream reported about its usage. + * + * Read by the canonical runner once its subscribing scope has unwound, so every + * destination it reads has been sealed by its own subscription's teardown. + * Undefined for a stream this middleware did not wrap, for one never + * subscribed, and for a turn whose provider reported nothing. + */ +export function* completedPromptUsage( + stream: Stream, +): Operation { + const audits = yield* PromptAudits.get(); + const audit = audits?.get(stream); + return audit === undefined ? undefined : canonicalUsage(audit.usage); } /** @@ -287,16 +343,32 @@ export function* completedPromptAudit( */ function audited( stream: Stream, - ledger: PromptAuditLedger, + audit: StreamAudit, ): Stream { + const ledger = audit.permissions.ledger; return { *[Symbol.iterator]() { + // One usage destination per subscription, because each subscription of + // a cold stream is its own native turn. + const usage = promptUsageDestination(); + audit.usage.push(usage); + // Registered before the provider's own subscription, so it runs after + // every finalizer that subscription registers in this same scope: a + // receiver stays open through the turn's joined teardown and is inert + // after it. + yield* ensure(() => { + usage.seal(); + }); // A `Stream` is itself the operation that subscribes, so this is that // subscription run beneath the ledger — not a copy of it. - const subscription = yield* PromptAudit.with(ledger, () => stream); + const subscription = yield* PromptAudit.with(ledger, () => + PromptUsageAudit.with(usage, () => stream), + ); return { *next() { - return yield* PromptAudit.with(ledger, () => subscription.next()); + return yield* PromptAudit.with(ledger, () => + PromptUsageAudit.with(usage, () => subscription.next()), + ); }, }; }, @@ -660,6 +732,9 @@ function serializePromptRecord(record: PromptRecord): Json { value: record.checkpoint.value, }; } + if (record.usage !== undefined) { + payload.usage = serializePromptUsage(record.usage); + } return payload; } @@ -813,6 +888,7 @@ export function parsePromptRecord(value: unknown): PromptRecord | undefined { permissions, raised, checkpoint, + usage, } = value; if ( typeof sequence !== "number" || @@ -870,5 +946,15 @@ export function parsePromptRecord(value: unknown): PromptRecord | undefined { } record.checkpoint = parsed; } + if (usage !== undefined) { + // Refused rather than read past: a present account of spend that does not + // read back is not one this build can report, and dropping it would make a + // damaged record look like one that never measured anything. + const read = readPromptUsage(usage); + if (read === undefined) { + return undefined; + } + record.usage = read; + } return record; } diff --git a/packages/core/src/agent/launch-coordinator.ts b/packages/core/src/agent/launch-coordinator.ts index 408c4b79c..5a6eeabe5 100644 --- a/packages/core/src/agent/launch-coordinator.ts +++ b/packages/core/src/agent/launch-coordinator.ts @@ -22,6 +22,8 @@ import { createSessionSetupOwner } from "./session-setup.ts"; import type { AgentSessionSetup, SessionSetupOwner } from "./session-setup.ts"; import type { ReadSessionUse } from "./session-use.ts"; import { associateCheckpoint } from "./checkpoint.ts"; +import { currentPromptUsageReporter } from "./journal.ts"; +import type { PromptUsageReporter } from "./usage.ts"; import { AgentLaunchProtocolError } from "./launch-request.ts"; import type { AgentLaunchRequest, IssuedLaunch } from "./launch-request.ts"; import { consumeSessionRequest } from "./session-request.ts"; @@ -124,6 +126,15 @@ export interface AgentLaunchCoordinator { * presents itself as a configured use and is not one refuses. */ sessionUse(routed: string | Session | undefined): ReadSessionUse | undefined; + /** + * Bind usage reporting to the current subscription's live audit, when available. + * + * Capture before native turn work and keep the receiver through that turn's + * joined cleanup. Reporting records observation only; it appends no journal + * event and establishes no canonical identity, checkpoint or successful outcome. + * An absent method or receiver leaves usage unavailable without refusing work. + */ + promptUsage?(): Operation; /** * Say how a session this provider just issued is configured. * @@ -244,6 +255,9 @@ export function createLaunchCoordinator( sessionUse(routed) { return owner.read(routed); }, + // Read from the subscription being started, never from the coordinator: + // one installation serves many turns, and each turn's receiver is its own. + promptUsage: currentPromptUsageReporter, *perform(request, phases) { const launch = locate(request); launch.issued.admit(request, generation); diff --git a/packages/core/src/agent/provider-api.ts b/packages/core/src/agent/provider-api.ts index 1f15219de..a9763994e 100644 --- a/packages/core/src/agent/provider-api.ts +++ b/packages/core/src/agent/provider-api.ts @@ -167,6 +167,9 @@ function deliveryOf(value: unknown): { const sessionSetup = Reflect.get(launchCoordinator, "sessionSetup"); const checkpoint = Reflect.get(launchCoordinator, "checkpoint"); const sessionUse = Reflect.get(launchCoordinator, "sessionUse"); + // Optional, and deliberately outside the check below: a coordinator that + // offers no usage observation still coordinates everything a turn needs. + const promptUsage = Reflect.get(launchCoordinator, "promptUsage"); const registerSessionConfiguration = Reflect.get( launchCoordinator, "registerSessionConfiguration", @@ -194,6 +197,9 @@ function deliveryOf(value: unknown): { Reflect.apply(checkpoint, launchCoordinator, [terminal, token]); }, sessionUse: (routed) => Reflect.apply(sessionUse, launchCoordinator, [routed]), + ...(typeof promptUsage === "function" + ? { promptUsage: () => Reflect.apply(promptUsage, launchCoordinator, []) } + : {}), }, }; } diff --git a/packages/core/src/agent/usage.ts b/packages/core/src/agent/usage.ts new file mode 100644 index 000000000..805f91e64 --- /dev/null +++ b/packages/core/src/agent/usage.ts @@ -0,0 +1,569 @@ +/** + * What one Prompt's turn reported about its own consumption + * (specs/acp-client-spec.md §Prompt usage). + * + * A provider observes the readings; this module owns what one of them means + * once it reaches Core. The value is descriptive and nothing else: no string in + * it routes a turn, authorizes work, names a Session or continues one. The + * canonical Prompt record keeps it as optional data beside the identity, + * Session and outcome the record already carries. + * + * Readings are source views, not a ledger. A response total, a quota total and + * per-model rows can describe overlapping work, so nothing here adds one to + * another, and coverage says how much of the outer Prompt the readings are + * known to describe. An empty list of readings is not zero, `tokens: null` is a + * source that said null, and an omitted number is unknown. + */ + +import type { Json } from "@executablemd/durable-streams"; +import type { Operation } from "effection"; +import { inputValues } from "../secrets/inputs.ts"; +import { secretPolicy, scanSecrets } from "../secrets/policy.ts"; + +/** One build identity, where it was observed. */ +export interface UsageBuild { + readonly version?: string; + readonly digest?: string; +} + +/** Which builds produced the turn the readings describe. */ +export interface UsageProvenance { + readonly host: "deno-source" | "compiled"; + readonly xmd: UsageBuild & { readonly sourceRevision?: string }; + readonly runtime: UsageBuild; + readonly adapter: UsageBuild; + readonly native?: UsageBuild; +} + +/** Counters a source reported, under the source's own field names. */ +export interface ReportedTokens { + readonly inputTokens?: number; + readonly outputTokens?: number; + readonly cachedReadTokens?: number; + readonly cachedInputTokens?: number; + readonly cachedWriteTokens?: number; + readonly thoughtTokens?: number; + readonly reasoningOutputTokens?: number; + readonly totalTokens?: number; +} + +interface UsageReadingBase { + readonly source: "response" | "quota-total" | "quota-model" | "update"; + readonly scope: "main-loop" | "last-request" | "broader-model-work" | "session" | "unknown"; + readonly model?: string; +} + +/** One source-labelled view: token counters, context occupancy, or reported cost. */ +export type UsageReading = UsageReadingBase & + ( + | { + readonly kind: "tokens"; + readonly counter: "turn-total" | "snapshot" | "increment" | "unknown"; + readonly tokens: ReportedTokens | null; + } + | { + readonly kind: "context"; + readonly used?: number; + readonly size?: number; + } + | { + readonly kind: "cost"; + readonly amount?: number; + readonly currency?: string; + } + ); + +/** How much of the outer Prompt the readings are known to describe. */ +export interface UsageCoverage { + readonly finalReport: "received" | "unavailable"; + readonly outerPrompt: "verified" | "partial" | "unknown"; + readonly limitations: readonly string[]; +} + +/** One turn's detached usage, as a provider reported it and a record retains it. */ +export interface PromptUsage { + readonly requestId?: string; + readonly providerRecordId?: string; + readonly providerMessageId?: string; + readonly providerSessionId?: string; + readonly provenance: UsageProvenance; + readonly readings: readonly UsageReading[]; + readonly coverage: UsageCoverage; +} + +/** Record a detached usage snapshot for the subscription that supplied this receiver. */ +export type PromptUsageReporter = (usage: PromptUsage) => void; + +/** A canonical turn whose stream was subscribed more than once has no one owner. */ +export const USAGE_ASSOCIATION_AMBIGUOUS = "association-ambiguous"; +/** An optional string failed the producing execution's disclosure checks. */ +export const USAGE_FIELD_WITHHELD = "field-withheld"; +/** A reported value did not read as the closed shape a record retains. */ +export const USAGE_INVALID = "invalid-usage"; + +const TOKEN_FIELDS: readonly (keyof ReportedTokens)[] = [ + "inputTokens", + "outputTokens", + "cachedReadTokens", + "cachedInputTokens", + "cachedWriteTokens", + "thoughtTokens", + "reasoningOutputTokens", + "totalTokens", +]; +const SOURCES: readonly UsageReading["source"][] = [ + "response", + "quota-total", + "quota-model", + "update", +]; +const SCOPES: readonly UsageReading["scope"][] = [ + "main-loop", + "last-request", + "broader-model-work", + "session", + "unknown", +]; +const COUNTERS = ["turn-total", "snapshot", "increment", "unknown"] as const; +const HOSTS: readonly UsageProvenance["host"][] = ["deno-source", "compiled"]; +const FINAL_REPORTS: readonly UsageCoverage["finalReport"][] = ["received", "unavailable"]; +const OUTER_PROMPT: readonly UsageCoverage["outerPrompt"][] = ["verified", "partial", "unknown"]; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isCount(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function member(values: readonly T[], value: unknown): T | undefined { + return values.find((candidate) => candidate === value); +} + +/** Every key of `value` is one of `allowed`. */ +function closed(value: Record, allowed: readonly string[]): boolean { + return Object.keys(value).every((key) => allowed.includes(key)); +} + +const INVALID = Symbol("invalid usage"); +type Read = T | typeof INVALID; + +function optionalString(value: unknown): Read { + if (value === undefined) { + return undefined; + } + return typeof value === "string" && value.length > 0 ? value : INVALID; +} + +function optionalCount(value: unknown): Read { + if (value === undefined) { + return undefined; + } + return isCount(value) ? value : INVALID; +} + +function readBuild(value: unknown, extra: readonly string[] = []): Read> { + if (!isRecord(value) || !closed(value, ["version", "digest", ...extra])) { + return INVALID; + } + const build: Record = {}; + for (const key of ["version", "digest", ...extra]) { + const read = optionalString(value[key]); + if (read === INVALID) { + return INVALID; + } + if (read !== undefined) { + build[key] = read; + } + } + return build; +} + +function readProvenance(value: unknown): Read { + if (!isRecord(value) || !closed(value, ["host", "xmd", "runtime", "adapter", "native"])) { + return INVALID; + } + const host = member(HOSTS, value.host); + const xmd = readBuild(value.xmd, ["sourceRevision"]); + const runtime = readBuild(value.runtime); + const adapter = readBuild(value.adapter); + const native = value.native === undefined ? undefined : readBuild(value.native); + if ( + host === undefined || + xmd === INVALID || + runtime === INVALID || + adapter === INVALID || + native === INVALID + ) { + return INVALID; + } + return { host, xmd, runtime, adapter, ...(native === undefined ? {} : { native }) }; +} + +function readTokens(value: unknown): Read { + if (value === null) { + return null; + } + if (!isRecord(value) || !closed(value, TOKEN_FIELDS)) { + return INVALID; + } + const tokens: Record = {}; + for (const field of TOKEN_FIELDS) { + const read = optionalCount(value[field]); + if (read === INVALID) { + return INVALID; + } + if (read !== undefined) { + tokens[field] = read; + } + } + return tokens; +} + +function readReading(value: unknown): Read { + if (!isRecord(value)) { + return INVALID; + } + const source = member(SOURCES, value.source); + const scope = member(SCOPES, value.scope); + const model = optionalString(value.model); + if (source === undefined || scope === undefined || model === INVALID) { + return INVALID; + } + const base = { source, scope, ...(model === undefined ? {} : { model }) }; + if (value.kind === "tokens") { + if (!closed(value, ["source", "scope", "model", "kind", "counter", "tokens"])) { + return INVALID; + } + const counter = member(COUNTERS, value.counter); + const tokens = readTokens(value.tokens); + if (counter === undefined || tokens === INVALID || !Object.hasOwn(value, "tokens")) { + return INVALID; + } + return { ...base, kind: "tokens", counter, tokens }; + } + if (value.kind === "context") { + if (!closed(value, ["source", "scope", "model", "kind", "used", "size"])) { + return INVALID; + } + const used = optionalCount(value.used); + const size = optionalCount(value.size); + if (used === INVALID || size === INVALID) { + return INVALID; + } + return { + ...base, + kind: "context", + ...(used === undefined ? {} : { used }), + ...(size === undefined ? {} : { size }), + }; + } + if (value.kind === "cost") { + if (!closed(value, ["source", "scope", "model", "kind", "amount", "currency"])) { + return INVALID; + } + const amount = optionalCount(value.amount); + const currency = optionalString(value.currency); + if (amount === INVALID || currency === INVALID) { + return INVALID; + } + return { + ...base, + kind: "cost", + ...(amount === undefined ? {} : { amount }), + ...(currency === undefined ? {} : { currency }), + }; + } + return INVALID; +} + +function readCoverage(value: unknown): Read { + if (!isRecord(value) || !closed(value, ["finalReport", "outerPrompt", "limitations"])) { + return INVALID; + } + const finalReport = member(FINAL_REPORTS, value.finalReport); + const outerPrompt = member(OUTER_PROMPT, value.outerPrompt); + const limitations = value.limitations; + if ( + finalReport === undefined || + outerPrompt === undefined || + !Array.isArray(limitations) || + !limitations.every((limitation) => typeof limitation === "string" && limitation.length > 0) + ) { + return INVALID; + } + return { finalReport, outerPrompt, limitations: [...limitations] }; +} + +/** + * One usage value read as the closed shape it describes, or nothing. + * + * Closed in both directions, like a permission audit: a member nothing here + * defines refuses the whole value rather than being read past, because a reader + * that skipped part of an account of spend would be reporting numbers it had + * not read. + */ +export function readPromptUsage(value: unknown): PromptUsage | undefined { + if ( + !isRecord(value) || + !closed(value, [ + "requestId", + "providerRecordId", + "providerMessageId", + "providerSessionId", + "provenance", + "readings", + "coverage", + ]) + ) { + return undefined; + } + const identities: Record = {}; + for (const key of ["requestId", "providerRecordId", "providerMessageId", "providerSessionId"]) { + const read = optionalString(value[key]); + if (read === INVALID) { + return undefined; + } + if (read !== undefined) { + identities[key] = read; + } + } + const provenance = readProvenance(value.provenance); + const coverage = readCoverage(value.coverage); + if (provenance === INVALID || coverage === INVALID || !Array.isArray(value.readings)) { + return undefined; + } + const readings: UsageReading[] = []; + for (const entry of value.readings) { + const reading = readReading(entry); + if (reading === INVALID) { + return undefined; + } + readings.push(reading); + } + return { ...identities, provenance, readings, coverage }; +} + +/** The value as the journal writes it: a fresh copy of exactly the members it read. */ +export function serializePromptUsage(usage: PromptUsage): Json { + // Read back through the parser first, so what is written is what a reader + // will accept and nothing a caller attached to the object comes along. + const parsed = readPromptUsage(usage); + if (parsed === undefined) { + throw new Error("a Prompt usage value does not read as the shape a record retains"); + } + return JSON.parse(JSON.stringify(parsed)); +} + +/** + * The receipt a canonical turn retains when what was reported cannot be. + * + * Provenance is kept because it says which builds produced the turn; every + * identity, reading and number is dropped, and the limitation says why. + */ +function unavailableUsage(provenance: UsageProvenance, limitation: string): PromptUsage { + return { + provenance, + readings: [], + coverage: { finalReport: "unavailable", outerPrompt: "unknown", limitations: [limitation] }, + }; +} + +/** + * One native subscription's place to report usage. + * + * Created when a stream is actually subscribed, never when it is wrapped: a + * cold stream subscribed twice starts two turns, and each turn's readings are + * its own. The receiver is a closure over this destination alone, so a + * provider holding it can reach no other turn's slot, and once the subscribing + * scope has torn down the receiver records nothing. + */ +export interface PromptUsageDestination { + readonly report: PromptUsageReporter; + /** Stop accepting reports. Called once the subscription's own teardown has joined. */ + seal(): void; + /** The latest report, if one arrived while the destination was open. */ + reported(): PromptUsage | undefined; +} + +export function promptUsageDestination(): PromptUsageDestination { + let open = true; + let latest: PromptUsage | undefined; + let invalid: UsageProvenance | undefined; + return { + report(usage: PromptUsage): void { + if (!open) { + return; + } + // Detached at the moment it arrives: a provider is free to keep mutating + // whatever it built, and the record describes what was reported. + const parsed = readPromptUsage(usage); + if (parsed === undefined) { + const provenance = readProvenance(Reflect.get(Object(usage), "provenance")); + invalid = provenance === INVALID ? invalid : provenance; + latest = undefined; + return; + } + invalid = undefined; + latest = parsed; + }, + seal(): void { + open = false; + }, + reported(): PromptUsage | undefined { + if (latest !== undefined) { + return latest; + } + return invalid === undefined ? undefined : unavailableUsage(invalid, USAGE_INVALID); + }, + }; +} + +/** + * The usage one canonical consumption retains, given every subscription made + * through the stream it consumed. + * + * One subscription is the turn. More than one means this stream started more + * than one native turn, and nothing here can say which of them the record + * describes, so the answer keeps the explicit gap rather than whichever turn + * reported last. + */ +export function canonicalUsage( + subscriptions: readonly PromptUsageDestination[], +): PromptUsage | undefined { + if (subscriptions.length === 0) { + return undefined; + } + if (subscriptions.length === 1) { + return subscriptions[0].reported(); + } + const reported = subscriptions.flatMap((subscription) => { + const usage = subscription.reported(); + return usage === undefined ? [] : [usage]; + }); + if (reported.length === 0) { + return undefined; + } + return unavailableUsage(reported[0].provenance, USAGE_ASSOCIATION_AMBIGUOUS); +} + +/** + * Whether one optional string may be retained by the execution running now. + * + * Protected inputs are refused whatever the scanner setting, because that check + * is not the scanner's to waive. Doubt withholds: an absent or unauthentic + * policy, a finding and a scanner that failed all answer no. + */ +function* disclosable(value: string): Operation { + try { + const inputs = yield* inputValues(); + if (inputs !== undefined) { + if (inputs.contains(value)) { + return false; + } + const shown = inputs.display(value); + if (!shown.ok || shown.value !== value) { + return false; + } + } + const policy = yield* secretPolicy(); + if (!policy.enabled) { + return true; + } + return (yield* scanSecrets(value)).length === 0; + } catch { + return false; + } +} + +/** + * The part of a reported value this execution may retain. + * + * Asked by the canonical runner, under the execution that produced the turn and + * before the record is serialized. Each optional string is checked on its own, + * so one unsafe model name withholds that name rather than the whole account; + * a reading whose only identity was a withheld model is dropped, because + * keeping its numbers without the model would let it read as another view. + * Numbers, enumerations and the fixed limitation names carry nothing a host + * wrote. The journal's own gate still decides whether the record is persisted. + */ +export function* retainablePromptUsage(usage: PromptUsage): Operation { + let withheld = false; + function* keep(value: string | undefined): Operation { + if (value === undefined) { + return undefined; + } + if (yield* disclosable(value)) { + return value; + } + withheld = true; + return undefined; + } + function* build( + value: T, + ): Operation> { + const kept: Record = {}; + for (const key of ["version", "digest", "sourceRevision"] as const) { + const read = yield* keep(value[key]); + if (read !== undefined) { + kept[key] = read; + } + } + return kept; + } + + const identities: Record = {}; + for (const key of [ + "requestId", + "providerRecordId", + "providerMessageId", + "providerSessionId", + ] as const) { + const read = yield* keep(usage[key]); + if (read !== undefined) { + identities[key] = read; + } + } + const provenance: UsageProvenance = { + host: usage.provenance.host, + xmd: yield* build(usage.provenance.xmd), + runtime: yield* build(usage.provenance.runtime), + adapter: yield* build(usage.provenance.adapter), + ...(usage.provenance.native === undefined + ? {} + : { native: yield* build(usage.provenance.native) }), + }; + const readings: UsageReading[] = []; + for (const reading of usage.readings) { + if (reading.model !== undefined) { + const model = yield* keep(reading.model); + if (model === undefined) { + continue; + } + } + if (reading.kind === "cost" && reading.currency !== undefined) { + const currency = yield* keep(reading.currency); + if (currency === undefined) { + readings.push({ + source: reading.source, + scope: reading.scope, + kind: "cost", + ...(reading.amount === undefined ? {} : { amount: reading.amount }), + }); + continue; + } + } + readings.push(reading); + } + const limitations = [...usage.coverage.limitations]; + if (withheld && !limitations.includes(USAGE_FIELD_WITHHELD)) { + limitations.push(USAGE_FIELD_WITHHELD); + } + return { + ...identities, + provenance, + readings, + coverage: { ...usage.coverage, limitations }, + }; +} diff --git a/packages/core/tests/agent-usage.test.ts b/packages/core/tests/agent-usage.test.ts new file mode 100644 index 000000000..b2d6dce98 --- /dev/null +++ b/packages/core/tests/agent-usage.test.ts @@ -0,0 +1,656 @@ +/** + * Tier PU — Prompt usage in Core (specs/acp-client-spec.md §Prompt usage). + * + * The retained shape and its reader, the subscription-local receiver the + * delivered coordinator hands an installed provider, and what the canonical + * runner keeps of it: authored Prompts and held conversations through the same + * path, replay without a provider, and direct calls that retain nothing. + */ + +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import type { DurableEvent } from "@executablemd/durable-streams"; +import { ensure, scoped, spawn, withResolvers } from "effection"; +import type { Operation, Result, Stream } from "effection"; +import { ensureDir, rm, writeTextFile } from "@effectionx/fs"; +import { randomUUID } from "node:crypto"; +import * as path from "node:path"; +import * as os from "node:os"; +import { Agent } from "../src/agent/agent-api.ts"; +import type { AgentPromptEvent, PromptOptions, Session } from "../src/agent/agent-api.ts"; +import { executeInstalled, useAgentConversation } from "../host.ts"; +import type { ExecutionInstallation } from "../host.ts"; +import { agentIdentityComponents, installAgentComponents } from "../src/agent/components.ts"; +import { registerAgentProvider } from "../src/agent/provider-api.ts"; +import type { AgentProviderFactory } from "../src/agent/provider-api.ts"; +import type { AgentLaunchCoordinator } from "../src/agent/launch-coordinator.ts"; +import { AgentConversationProvider } from "../src/agent/conversation-provider.ts"; +import type { AgentSessionRequest } from "../src/agent/session-request.ts"; +import type { ProviderSessionConfiguration } from "../src/agent/session-setup.ts"; +import { + completedPromptUsage, + observePermissionDecisions, + parsePromptRecord, + readCompletedPrompts, +} from "../src/agent/journal.ts"; +import { + promptUsageDestination, + readPromptUsage, + serializePromptUsage, +} from "../src/agent/usage.ts"; +import { createLaunchCoordinator } from "../src/agent/launch-coordinator.ts"; +import { registerComponents } from "../src/components/registration.ts"; +import type { SecretScanner } from "../src/secrets/scanner.ts"; +import type { SecretFinding } from "../src/secrets/findings.ts"; +import type { PromptUsage, PromptUsageReporter } from "../src/agent/usage.ts"; +import { useSecretScannerFactory } from "../src/secrets/policy.ts"; +import { createSecretScanner } from "../src/secrets/scanner.ts"; +import type { Json } from "../src/types.ts"; + +const AGENT = "usage-agent"; + +/** The constrained configuration a held conversation's provider states. */ +const HELD_CONFIGURATION: ProviderSessionConfiguration = Object.freeze({ + providerTools: Object.freeze([]), + mcpServers: Object.freeze([]), + permissions: Object.freeze({ + permissionMode: "deny-all", + routing: "strict", + nonInteractivePermissions: "deny", + }), + systemPrompt: null, +}); + +/** A complete value, with an explicit zero, a source null and absent counters. */ +function sampleUsage(label: string, model = "model-a"): PromptUsage { + return { + requestId: `request-${label}`, + providerRecordId: `record-${label}`, + providerMessageId: `message-${label}`, + providerSessionId: `session-${label}`, + provenance: { + host: "deno-source", + xmd: { version: "0.0.0-test", sourceRevision: "abc123" }, + runtime: { version: "0.21.1" }, + adapter: { version: "9.9.9", digest: "sha256:00" }, + native: { version: "1.2.3" }, + }, + readings: [ + { + source: "response", + scope: "main-loop", + kind: "tokens", + counter: "turn-total", + tokens: { inputTokens: 10, outputTokens: 0, totalTokens: 10 }, + }, + { + source: "quota-total", + scope: "main-loop", + kind: "tokens", + counter: "turn-total", + tokens: null, + }, + { + source: "quota-model", + scope: "broader-model-work", + model, + kind: "tokens", + counter: "turn-total", + tokens: { cachedInputTokens: 4, reasoningOutputTokens: 0 }, + }, + { + source: "update", + scope: "session", + kind: "context", + used: 120, + size: 1000, + }, + { + source: "update", + scope: "unknown", + kind: "cost", + amount: 0.5, + currency: "USD", + }, + ], + coverage: { + finalReport: "received", + outerPrompt: "partial", + limitations: ["fixture-limit"], + }, + }; +} + +/** How the stub provider answers one subscribed turn. */ +interface TurnPlan { + /** Report this usage from inside the subscribed turn, before its terminal. */ + usage?: () => PromptUsage | undefined; + /** The terminal status this turn ends with. */ + status?: "completed" | "failed" | "cancelled"; + /** Hold the turn after it starts until this resolves. */ + hold?: Operation; + /** Called once the turn's receiver is captured. */ + captured?: (reporter: PromptUsageReporter | undefined) => void; +} + +interface StubTrace { + /** How many times a turn was subscribed: one native turn each. */ + subscriptions: number; + /** How many receivers were captured. */ + acquisitions: number; +} + +function newTrace(): StubTrace { + return { subscriptions: 0, acquisitions: 0 }; +} + +/** One cold turn. The plan is chosen per subscription, because each is its own turn. */ +function stubTurn( + content: string, + options: PromptOptions | undefined, + coordinator: AgentLaunchCoordinator, + trace: StubTrace, + plans: () => TurnPlan, +): Stream { + return { + *[Symbol.iterator]() { + trace.subscriptions++; + const plan = plans(); + let report: PromptUsageReporter | undefined; + if (coordinator.promptUsage !== undefined) { + trace.acquisitions++; + report = yield* coordinator.promptUsage(); + } + plan.captured?.(report); + const session: Session = + typeof options?.session === "object" && options.session !== null + ? options.session + : { + sessionKey: `stub:${options?.session ?? "default"}`, + cwd: "/stub", + }; + const events: AgentPromptEvent[] = [ + { type: "started", agent: options?.agent ?? AGENT, session }, + { type: "text_delta", text: `[${content}]` }, + { type: "terminal", status: plan.status ?? "completed" }, + ]; + let index = 0; + let held = false; + return { + *next() { + if (index === 1 && plan.hold !== undefined && !held) { + held = true; + yield* plan.hold; + } + if (index === 2) { + const usage = plan.usage?.(); + if (usage !== undefined) { + report?.(usage); + } + } + if (index < events.length) { + return { done: false, value: events[index++]! }; + } + return { done: true, value: `[${content}]` }; + }, + }; + }, + }; +} + +function stubFactory(trace: StubTrace, plans: () => TurnPlan): AgentProviderFactory { + return function* (_options, coordinator) { + yield* Agent.around( + { + // deno-lint-ignore require-yield + *agent([name]) { + return name ?? AGENT; + }, + // deno-lint-ignore require-yield + *session([name]) { + return { + sessionKey: `stub:${typeof name === "string" ? name : "default"}`, + cwd: "/stub", + }; + }, + // deno-lint-ignore require-yield + *prompt([content, options]) { + return stubTurn(content, options, coordinator, trace, plans); + }, + }, + { at: "min" }, + ); + yield* AgentConversationProvider.set({ + // deno-lint-ignore require-yield + *agent(name: string): Operation { + return name; + }, + *session(request: AgentSessionRequest, agent: string): Operation { + const setup = coordinator.sessionSetup(request); + const authorized = yield* setup.activate("stub", "stub-cmd", HELD_CONFIGURATION); + if (!authorized.ok) { + throw authorized.error; + } + return yield* setup.finish( + { sessionKey: `stub:${request.name ?? agent}`, cwd: "/stub" }, + { kind: "new" }, + ); + }, + // deno-lint-ignore require-yield + *options(agent: string) { + return { agent, model: null, effort: null }; + }, + // deno-lint-ignore require-yield + *prompt(content: string, options: PromptOptions & { agent: string; session: Session }) { + return stubTurn(content, options, coordinator, trace, plans); + }, + }); + }; +} + +function always(plan: TurnPlan): () => TurnPlan { + return () => ({ ...plan }); +} + +function* runDocument( + doc: string, + stream: InMemoryStream, + factory: AgentProviderFactory, + options: { + register?: boolean; + direct?: boolean; + scanner?: () => SecretScanner; + } = {}, +): Operation<{ result: Result; events: DurableEvent[] }> { + const dir = path.join(os.tmpdir(), `xmd-pu-test-${randomUUID()}`); + yield* ensureDir(dir); + return yield* scoped(function* () { + yield* ensure(() => rm(dir, { recursive: true, force: true })); + const docPath = path.join(dir, "doc.md"); + yield* writeTextFile(docPath, doc); + if (options.register === true) { + yield* registerAgentProvider("usage-stub", factory); + yield* installAgentComponents({ + defaultAgent: AGENT, + permissionMode: "deny-all", + }); + } else { + yield* installAgentComponents({ + rootProvider: { + factory, + options: { defaultAgent: AGENT, permissionMode: "deny-all" }, + }, + }); + } + yield* useSecretScannerFactory(options.scanner ?? createSecretScanner); + if (options.direct === true) { + yield* registerComponents([ + { + name: "Direct", + origin: "test", + props: { + type: "object", + properties: {}, + additionalProperties: false, + }, + // A repository component asking the public Api directly: no + // authors this turn, so nothing may record it. + *fn() { + return yield* consume(yield* Agent.operations.prompt("direct", {})); + }, + }, + ]); + } + const execution = yield* executeInstalled( + { + path: docPath, + stream, + includes: [dir], + }, + [{ components: agentIdentityComponents() }], + ); + const output = yield* execution.output; + let next = yield* output.next(); + while (!next.done) { + next = yield* output.next(); + } + return { result: yield* execution, events: yield* stream.readAll() }; + }); +} + +function promptValues(events: readonly DurableEvent[]): Record[] { + return events.flatMap((event) => { + if (event.type !== "yield" || event.description.type !== "agent_prompt") { + return []; + } + if (event.result.status !== "ok") { + return []; + } + const value = event.result.value; + return typeof value === "object" && value !== null && !Array.isArray(value) + ? [value as Record] + : []; + }); +} + +function versioned(version: string): PromptUsage { + const usage = sampleUsage(version); + return { ...usage, provenance: { ...usage.provenance, xmd: { version } } }; +} + +function* consume(stream: Stream): Operation { + return yield* scoped(function* () { + const subscription = yield* stream; + let next = yield* subscription.next(); + while (!next.done) { + next = yield* subscription.next(); + } + return next.value; + }); +} + +describe("Tier PU — subscription-local receiver", () => { + it("PU-B1: a sealed destination ignores reports; an unreadable report keeps only its provenance", function* () { + const destination = promptUsageDestination(); + destination.report(versioned("first")); + destination.seal(); + destination.report(versioned("late")); + expect(destination.reported()?.provenance.xmd.version).toBe("first"); + + const broken = promptUsageDestination(); + broken.report({ + ...versioned("broken"), + readings: [{ bogus: true } as never], + }); + expect(broken.reported()).toEqual({ + provenance: versioned("broken").provenance, + readings: [], + coverage: { + finalReport: "unavailable", + outerPrompt: "unknown", + limitations: ["invalid-usage"], + }, + }); + }); + + it("PU-B2: a cold stream captures nothing; concurrent subscriptions keep their own receivers", function* () { + const trace = newTrace(); + const firstCaptured = withResolvers(); + const firstRelease = withResolvers(); + const receivers: (PromptUsageReporter | undefined)[] = []; + const plans: TurnPlan[] = [ + { + captured: (report) => { + receivers.push(report); + firstCaptured.resolve(); + }, + hold: firstRelease.operation, + usage: () => versioned("first"), + }, + { + captured: (report) => receivers.push(report), + usage: () => versioned("second"), + }, + ]; + yield* scoped(function* () { + yield* observePermissionDecisions(); + const coordinator = createLaunchCoordinator({}, () => []); + yield* stubFactory(trace, () => plans.shift() ?? {})( + { defaultAgent: AGENT, permissionMode: "deny-all" }, + coordinator, + ); + const cold = yield* Agent.operations.prompt("twice", {}); + // Wrapping and obtaining the stream starts nothing and captures nothing. + expect(trace.acquisitions).toBe(0); + expect(trace.subscriptions).toBe(0); + + const first = yield* spawn(() => consume(cold)); + yield* firstCaptured.operation; + expect(yield* consume(cold)).toBe("[twice]"); + // The second subscription has ended: its receiver is inert, and the + // first — still running — is unaffected by that teardown. + receivers[1]?.(versioned("late")); + firstRelease.resolve(); + expect(yield* first).toBe("[twice]"); + expect(receivers.length).toBe(2); + expect(receivers[0]).not.toBe(receivers[1]); + + // Two native turns through one stream: no canonical owner, so the + // association stays an explicit gap rather than the last report. + const canonical = yield* completedPromptUsage(cold); + expect(canonical?.readings).toEqual([]); + expect(canonical?.coverage.limitations).toEqual(["association-ambiguous"]); + expect(canonical?.provenance.xmd.version).toBe("first"); + }); + expect(trace.subscriptions).toBe(2); + expect(trace.acquisitions).toBe(2); + }); + + it("PU-B3: a direct Agent.prompt() gets an ephemeral receiver and lends nothing to the next Prompt", function* () { + const trace = newTrace(); + const receivers: (PromptUsageReporter | undefined)[] = []; + const plans: TurnPlan[] = [ + { + usage: () => versioned("direct"), + captured: (report) => receivers.push(report), + }, + { captured: (report) => receivers.push(report) }, + { + usage: () => versioned("canonical"), + captured: (report) => receivers.push(report), + }, + ]; + const { result, events } = yield* runDocument( + "\n\nsilent\n\nmeasured\n", + new InMemoryStream(), + stubFactory(trace, () => plans.shift() ?? {}), + { direct: true }, + ); + expect(result.ok).toBe(true); + expect(receivers.length).toBe(3); + expect(receivers[0]).not.toBe(undefined); + // The direct call's turn reported, and no record exists for it; the + // adjacent Prompt that reported nothing retains nothing; the one that did + // retains its own. + const records = promptValues(events); + expect(records.length).toBe(2); + expect(Object.hasOwn(records[0], "usage")).toBe(false); + expect(parsePromptRecord(records[1])?.usage?.provenance.xmd.version).toBe("canonical"); + }); + + it("PU-B4: a registered provider's delivered coordinator keeps the optional receiver", function* () { + const trace = newTrace(); + const { result, events } = yield* runDocument( + '\n\none\n\n\n', + new InMemoryStream(), + stubFactory(trace, always({ usage: () => versioned("registered") })), + { register: true }, + ); + expect(result.ok).toBe(true); + expect(trace.acquisitions).toBe(1); + expect(parsePromptRecord(promptValues(events)[0])?.usage?.provenance.xmd.version).toBe( + "registered", + ); + }); +}); + +describe("Tier PU — canonical retention and replay", () => { + it("PU-P1: an authored Prompt retains usage, and a completed replay restores it without a turn", function* () { + const stream = new InMemoryStream(); + const live = newTrace(); + const first = yield* runDocument( + "one\n", + stream, + stubFactory(live, always({ usage: () => sampleUsage("one") })), + ); + expect(first.result.ok).toBe(true); + const recorded = promptValues(first.events); + expect(recorded.length).toBe(1); + expect(parsePromptRecord(recorded[0])?.usage).toEqual(sampleUsage("one")); + + const replayTrace = newTrace(); + const again = yield* runDocument( + "one\n", + stream, + stubFactory(replayTrace, always({ usage: () => versioned("refreshed") })), + ); + expect(again.result.ok).toBe(true); + expect(replayTrace.subscriptions).toBe(0); + expect(replayTrace.acquisitions).toBe(0); + expect(again.events.length).toBe(first.events.length); + const restored = yield* readCompletedPrompts(stream); + expect(restored?.map((record) => record.usage)).toEqual([sampleUsage("one")]); + }); + + it("PU-P2: an acknowledged failed turn keeps what it reported, and no checkpoint", function* () { + const failing = sampleUsage("failed"); + const partial: PromptUsage = { + ...failing, + coverage: { + finalReport: "unavailable", + outerPrompt: "unknown", + limitations: ["final-report-unavailable"], + }, + }; + const { events } = yield* runDocument( + "fails\n", + new InMemoryStream(), + stubFactory(newTrace(), always({ status: "failed", usage: () => partial })), + ); + const record = parsePromptRecord(promptValues(events)[0]); + expect(record?.status).toBe("failed"); + expect(record?.usage).toEqual(partial); + expect(record?.checkpoint).toBe(undefined); + }); + + it("PU-P3: a held conversation's turn retains usage through the same runner", function* () { + const stream = new InMemoryStream(); + const trace = newTrace(); + const factory = stubFactory(trace, always({ usage: () => versioned("held") })); + const installation: ExecutionInstallation = { + *install(): Operation { + yield* installAgentComponents({ + rootProvider: { + factory, + options: { defaultAgent: AGENT, permissionMode: "deny-all" }, + }, + }); + }, + }; + const answered = yield* scoped(function* () { + const opened = yield* useAgentConversation({ + history: stream, + id: "chat-usage", + agent: AGENT, + installations: [installation], + }); + if (!opened.ok) { + throw opened.error; + } + return yield* opened.value.prompt("hello"); + }); + expect(answered.ok ? "ok" : answered.error.message).toBe("ok"); + const records = promptValues(yield* stream.readAll()); + expect(records.length).toBe(1); + expect(parsePromptRecord(records[0])?.usage?.provenance.xmd.version).toBe("held"); + expect(trace.subscriptions).toBe(1); + }); + + it("PU-P4: an unsafe model name is withheld by the producing execution; the rest is kept", function* () { + const marker = ["tok", "en-", "USAGE01"].join(""); + const unsafe = sampleUsage("unsafe", marker); + const { result, events } = yield* scoped(function* () { + return yield* runDocument( + "one\n", + new InMemoryStream(), + stubFactory(newTrace(), always({ usage: () => unsafe })), + { scanner: markerScanner(marker) }, + ); + }); + expect(result.ok).toBe(true); + const retained = parsePromptRecord(promptValues(events)[0])?.usage; + expect(JSON.stringify(retained).includes(marker)).toBe(false); + expect(retained?.readings.some((reading) => reading.source === "quota-model")).toBe(false); + expect(retained?.readings.length).toBe(unsafe.readings.length - 1); + expect(retained?.coverage.limitations).toEqual(["fixture-limit", "field-withheld"]); + expect(retained?.requestId).toBe(unsafe.requestId); + }); +}); + +/** A scanner that finds a secret in whatever text contains `marker`. */ +function markerScanner(marker: string): () => SecretScanner { + return () => ({ + // deno-lint-ignore require-yield + *scan(content: string): Operation { + return content.includes(marker) + ? [ + { + ruleId: "test-rule", + messageId: "TEST_SECRET", + location: { line: 1, column: 1 }, + fingerprint: "test-fingerprint", + }, + ] + : []; + }, + }); +} + +describe("Tier PU — retained data", () => { + it("PU-D1: a complete value round-trips, keeping zero, null and absent apart", function* () { + const usage = sampleUsage("one"); + const read = readPromptUsage(serializePromptUsage(usage)); + expect(read).toEqual(usage); + const [response, quota, model] = read?.readings ?? []; + expect(response?.kind === "tokens" ? response.tokens : "missing").toEqual({ + inputTokens: 10, + outputTokens: 0, + totalTokens: 10, + }); + expect(quota?.kind === "tokens" ? quota.tokens : "missing").toBe(null); + expect( + model?.kind === "tokens" && + model.tokens !== null && + Object.hasOwn(model.tokens, "inputTokens"), + ).toBe(false); + }); + + it("PU-D2: present malformed usage refuses the record; absent usage stays unknown", function* () { + const base = { + sequence: 0, + agent: AGENT, + sessionKey: "stub:default", + status: "completed", + text: "", + }; + expect(parsePromptRecord(base)).not.toBe(undefined); + expect(parsePromptRecord(base)?.usage).toBe(undefined); + const written = serializePromptUsage(sampleUsage("one")); + if (typeof written !== "object" || written === null || Array.isArray(written)) { + throw new Error("usage serializes as an object"); + } + expect(parsePromptRecord({ ...base, usage: written })?.usage).toEqual(sampleUsage("one")); + const reading = { + source: "response", + scope: "main-loop", + kind: "tokens", + counter: "turn-total", + }; + for (const malformed of [ + { ...written, extra: 1 }, + { ...written, readings: [{ ...reading, tokens: { inputTokens: -1 } }] }, + { ...written, readings: [{ ...reading, tokens: { inputTokens: 1, renamed: 2 } }] }, + { ...written, readings: [reading] }, + { + ...written, + coverage: { finalReport: "received", outerPrompt: "complete", limitations: [] }, + }, + { ...written, provenance: { host: "node", xmd: {}, runtime: {}, adapter: {} } }, + "usage", + null, + ]) { + expect(parsePromptRecord({ ...base, usage: malformed })).toBe(undefined); + } + }); +}); diff --git a/packages/workflow/tests/workflow-agent-checkpoints.test.ts b/packages/workflow/tests/workflow-agent-checkpoints.test.ts index 64d12dd34..30965c84c 100644 --- a/packages/workflow/tests/workflow-agent-checkpoints.test.ts +++ b/packages/workflow/tests/workflow-agent-checkpoints.test.ts @@ -16,7 +16,7 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { executeInstalled } from "@executablemd/core/host"; +import { executeInstalled, parsePromptRecord } from "@executablemd/core/host"; import { useAgentPromptPublisher } from "@executablemd/core/host"; import { Agent, @@ -30,6 +30,7 @@ import type { AgentLaunchCoordinator, AgentProviderFactory, Json, + PromptUsage, Session, } from "@executablemd/core"; import type { DurableEvent } from "@executablemd/durable-streams"; @@ -91,6 +92,34 @@ interface Reply { readonly failed?: true; /** Held here before the turn answers, so a run can be interrupted mid-turn. */ readonly hold?: () => Operation; + /** Reported through this subscription's usage receiver before the turn ends. */ + readonly usage?: PromptUsage; +} + +/** + * The same events, through a subscription that reports usage the way a + * provider does: it captures its receiver when subscribed, from the + * coordinator it was delivered, and reports before the turn's terminal. + */ +function reporting( + inner: Stream, + launchCoordinator: AgentLaunchCoordinator, + usage: PromptUsage | undefined, +): Stream { + return { + *[Symbol.iterator]() { + const report = yield* launchCoordinator.promptUsage?.() ?? none(); + if (usage !== undefined) { + report?.(usage); + } + return yield* inner; + }, + }; +} + +// deno-lint-ignore require-yield +function* none(): Operation { + return undefined; } function subscriptionOver( @@ -160,13 +189,17 @@ function scriptedProvider(replies: readonly Reply[]): AgentProviderFactory { value: reply.turnId, }); } - return subscriptionOver( - [ - { type: "started", agent: "codex", session }, - { type: "text_delta", text: reply.text }, - terminal, - ], - reply.text, + return reporting( + subscriptionOver( + [ + { type: "started", agent: "codex", session }, + { type: "text_delta", text: reply.text }, + terminal, + ], + reply.text, + ), + launchCoordinator, + reply.usage, ); }, // deno-lint-ignore require-yield @@ -723,4 +756,80 @@ describe("Tier WPC — retained Prompt checkpoints", () => { expect(declaredSchema(path)).toEqual(before); expect(committedPrompts(path)).toEqual([]); }); + + it("WPC13: a completed Prompt's usage commits with the Prompt and its association", function* () { + const root = yield* useStorageRoot(); + const path = runPath(root, "release-1.4"); + yield* withStorage(root, function* () { + const database = yield* createRun(); + yield* retainSession(database); + const attempt = yield* runDocument(database, ONE_PROMPT, [ + { text: "looks fine", turnId: "turn-alpha", usage: USAGE }, + ]); + expect(attempt.failure).toBe(undefined); + expect(retainedUsage(attempt.events)).toEqual([USAGE]); + expect(committedPrompts(path)).toHaveLength(1); + expect(yield* associations(database)).toHaveLength(1); + }); + }); + + it("WPC14: an unsuccessful Prompt keeps its usage and supplies no association", function* () { + const root = yield* useStorageRoot(); + const path = runPath(root, "release-1.4"); + yield* withStorage(root, function* () { + const database = yield* createRun(); + yield* retainSession(database); + const attempt = yield* runDocument(database, ONE_PROMPT, [ + // No turn id: a provider naming a failed turn is refused before the + // turn exists, which is WPC5's case rather than this one. + { text: "partial", failed: true, usage: USAGE }, + ]); + expect(retainedUsage(attempt.events)).toEqual([USAGE]); + expect(committedPrompts(path)).toHaveLength(1); + expect(committedAssociations(path)).toEqual([]); + }); + }); + + it("WPC15: a refused association rolls back the Prompt and the usage it carried", function* () { + const root = yield* useStorageRoot(); + const path = runPath(root, "release-1.4"); + yield* withStorage(root, function* () { + const database = yield* createRun(); + yield* retainSession(database); + refuseAssociation(path); + const attempt = yield* runDocument(database, ONE_PROMPT, [ + { text: "ok", turnId: "turn-alpha", usage: USAGE }, + ]); + expect(attempt.failure).toContain("the run refuses this association"); + expect(committedPrompts(path)).toEqual([]); + expect(committedAssociations(path)).toEqual([]); + expect(retainedUsage(yield* database.journal.readAll())).toEqual([]); + }); + }); }); + +/** One provider-neutral usage value, as a provider would report it. */ +const USAGE: PromptUsage = { + requestId: "request-wpc", + providerMessageId: "message-wpc", + provenance: { host: "deno-source", xmd: { version: "0.0.0-test" }, runtime: {}, adapter: {} }, + readings: [ + { + source: "response", + scope: "last-request", + kind: "tokens", + counter: "snapshot", + tokens: { inputTokens: 5, outputTokens: 1 }, + }, + ], + coverage: { finalReport: "received", outerPrompt: "partial", limitations: ["fixture"] }, +}; + +/** The usage each retained Prompt carries, in journal order. */ +function retainedUsage(events: readonly DurableEvent[]): (PromptUsage | undefined)[] { + return prompts(events).flatMap((event) => + event.type === "yield" && event.result.status === "ok" + ? [parsePromptRecord(event.result.value)?.usage] + : [], + ); +} diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index acd648304..143013eef 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -35,6 +35,12 @@ export interface RuntimeExclusion { const DERIVED_SCOPE = "https://github.com/taras/executable.md/issues/144"; const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ + { + path: "packages/cli/tests/agent-usage-calibration.test.ts", + reason: + "runs examples/agent-usage/calibrate.ts as `deno run` and its source host as the Deno CLI entrypoint; the subject is that Deno-hosted calibration driver, and the Node and Bun shards have no `deno` on PATH", + issue: DERIVED_SCOPE, + }, { path: "scripts/tests/build-npm.test.ts", reason: @@ -461,6 +467,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ "runs a document with two same-named sites against a real node:sqlite WorkflowRun database, with the run's Workspace attached and the mapping table read back across attachments; the store and the DOFS Workspace under it are both the Deno adapter's", issue: "https://github.com/taras/executable.md/issues/367", }, + { + path: "packages/cli/tests/prompt-usage-hosts.test.ts", + reason: + "its workflow row publishes a real Prompt into a real node:sqlite WorkflowRun database through the Deno storage adapter and the run's DOFS Workspace, beside the other host assemblies that row shares a file with", + issue: "https://github.com/taras/executable.md/issues/367", + }, { path: "packages/workflow/tests/workflow-agent-checkpoints.test.ts", reason: diff --git a/scripts/tests/acpx-vendor.test.ts b/scripts/tests/acpx-vendor.test.ts index 3e3759351..7bb786c3a 100644 --- a/scripts/tests/acpx-vendor.test.ts +++ b/scripts/tests/acpx-vendor.test.ts @@ -39,6 +39,7 @@ const SEAMS: readonly string[] = [ "strictResume", "Strict explicit resume:", "XMD prompt diagnostic:", + "XMD prompt usage:", ]; /** How far an introduced line may sit from one that names the seam. */ diff --git a/specs/acp-client-spec.md b/specs/acp-client-spec.md index 2381eb53b..bef811a45 100644 --- a/specs/acp-client-spec.md +++ b/specs/acp-client-spec.md @@ -939,7 +939,8 @@ successful. Each prompt is one durable operation (`agent_prompt`). The record carries the prompt's identity and input, the agent and session identity, terminal status, stop reason, text (including partial text on failure), any structured failure, -one optional canonical `configuration`, and one optional list of `permissions`. +one optional canonical `configuration`, one optional list of `permissions`, and +one optional `usage` (see [Prompt usage](#prompt-usage)). The configuration is retained only after the provider emits `started`: a turn that started is one the provider put under exactly those settings, while a configuration refusal started no turn and retains none. A prompt that starts @@ -1061,6 +1062,136 @@ that was thrown through `throwOnError` is marked `raised`; a partial replay re-throws it, while a full replay omits it from aggregate restoration because it was already handled where it occurred. +### Prompt usage + +An acknowledged Prompt keeps the usage its provider reported for that turn in +its own record. Selecting retained history — `--journal`, a workflow run, a REPL +entry, a Sidekick conversation — selects it; there is no accounting flag and no +second usage journal. A transient history stays transient. + +```json +{ + "usage": { + "requestId": "…", + "providerRecordId": "…", + "providerMessageId": "…", + "providerSessionId": "…", + "provenance": { + "host": "deno-source", + "xmd": { "version": "0.14.1" }, + "runtime": { "version": "0.21.1" }, + "adapter": { "version": "1.6.2", "digest": "sha256:…" } + }, + "readings": [ + { + "source": "response", + "scope": "last-request", + "kind": "tokens", + "counter": "snapshot", + "tokens": { "inputTokens": 900, "cachedReadTokens": 3000, "outputTokens": 40 } + }, + { "source": "update", "scope": "session", "kind": "context", "used": 12000, "size": 258000 } + ], + "coverage": { + "finalReport": "received", + "outerPrompt": "partial", + "limitations": ["codex-last-request-only", "codex-reasoning-inclusion-unverified"] + } + } +} +``` + +`requestId` is the runtime request this turn was started with; +`providerMessageId` is the user message the runtime generated and saved for it, +under the record `providerRecordId`; `providerSessionId` is the ACP session it +was prompted in. They are observations: none routes a turn, authorizes work or +names a native Session, which stays `agentSessionId`. Each is absent when it was +not observed. `provenance` names the host and the builds that produced the turn; +an unmeasured build stays absent rather than borrowed from elsewhere. + +Each reading is one source's own view, under the source's own counter names: + +| `source` | `kind` | Allowed values | +| --- | --- | --- | +| `response` | `tokens` | `inputTokens`, `outputTokens`, `cachedReadTokens`, `cachedWriteTokens`, `thoughtTokens`, `totalTokens` | +| `quota-total` | `tokens` | `totalTokens`, `inputTokens`, `cachedInputTokens`, `cachedWriteTokens`, `outputTokens`, `reasoningOutputTokens` | +| `quota-model` | `tokens` | the same counters, with the reported `model` | +| `update` | `tokens`, `context`, `cost` | live breakdown counters; occupancy `used` and `size`; reported `amount` and `currency` | + +Views of the same work are not additional spend: nothing adds a response total, +a quota total and per-model rows together. An omitted counter is unknown, +`tokens: null` is a source that reported null, and an empty list of readings is +not zero. Live updates replace one another, and an occupancy-only update leaves +an earlier cost or breakdown in place. `scope` and `counter` state what the +traced adapter release's view covers: + +| Adapter | `response` and `quota-total` | `quota-model` | Outer Prompt | +| --- | --- | --- | --- | +| Claude 0.70.0 | `main-loop`, `turn-total` | `broader-model-work`, `turn-total` | `partial`: subagent inclusion is unverified, absent counters are reported as zero, and reasoning is billed inside output | +| Codex 1.6.2 | `last-request`, `snapshot` | `last-request`, `snapshot` | `partial`: only the final internal request is reported | + +Any other release reports `unknown` scope and counter with +`source-semantics-unknown`. `finalReport` is `unavailable` when the turn ended +without a response view, and an unknown remainder stays unknown. +`examples/agent-usage/README.md` measures these meanings on a selected build. + +A failed or cancelled Prompt whose record is acknowledged keeps what was reported +before it ended, and never gains a checkpoint. Work stopped before its record was +acknowledged — a scope cancellation, a refused publication or append, a lost +process — leaves only the earlier history; nothing appends a substitute ending, +an invented zero or a second paid turn. A direct `Agent.prompt()` call, which no +canonical Prompt authors, may be observed but is retained nowhere and lends its +usage to no neighbouring Prompt. Native-launch materialization keeps its own +`MaterializationUsage` on its own record. + +Core selects the retained value under the producing execution's protection +before the record is serialized. Each optional string — identities, build +versions and digests, model names, currency — is withheld on its own when it +fails the secret scanner or matches a protected input, which a scanner opt-out +does not waive; `field-withheld` then joins the limitations. The complete event +still crosses the unchanged journal gate. + +The current reader validates a present `usage` completely and refuses the record +when it is malformed or carries an unknown member. A record without `usage` — +including every record written before it existed — reads as unknown and is never +backfilled. Completed replay returns the retained value and limitations without +reaching a provider, refreshing a measurement or recording more spend. + +#### How a provider reports it + +The coordinator Core delivers to an installed provider carries one optional +member: + +```ts +export type PromptUsageReporter = (usage: PromptUsage) => void; +promptUsage?(): Operation; +``` + +A provider calls it once when its stream is actually subscribed, before any +native work, and keeps the receiver for that turn. The receiver belongs to that +one subscription: Core opens it when the subscription starts, keeps it open +through the subscription's joined teardown and seals it afterwards, so a +retained receiver records nothing later and two subscriptions of one cold stream +never share one. A canonical turn whose stream was subscribed more than once +retains only an `association-ambiguous` receipt. Reporting is synchronous +observation: it appends nothing and establishes no identity, checkpoint or +outcome. An absent member, an undefined receiver and a report that never came +all leave usage unknown and refuse nothing. + +The same delivered coordinator carries an optional `promptUsage()` receiver, the one +route by which a provider's turn readings reach the Prompt that consumed them. Core +creates one usage destination per actual stream subscription inside the audited +stream that already carries the permission ledger, opens it before the provider's own +subscription and seals it after that subscription's teardown joins; the canonical +runner reads it afterwards, under the producing execution's disclosure, and the +existing `agent_prompt` append retains it. A receiver is observation only: it appends +nothing, authorizes nothing, and its absence or failure leaves execution unchanged. + +The ACPX provider reports from its one reader of the turn, outside its failure +handling: an acquisition, normalization or receiver failure leaves the turn's +events, text, outcome and cleanup exactly as they were. A provider installed +without a host's usage facts (`promptUsageHost`) reports nothing. + A native session launch is not a prompt and does not reuse `agent_prompt`. It retains its own `agent_session_launch` records, one per completed phase, so an interrupted launch resumes the provider session it already prepared rather than From b68cb728cf8a386b7f4331cd1cf3b18f9850fa4c Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 06:34:21 -0500 Subject: [PATCH 02/10] fix(examples): classify Codex subagents and unaccounted cancellation in usage calibration The Codex rollout reader now counts spawn_agent calls as native subagent work, so a Prompt that delegated is no longer reported as having no subagent. A run whose submitted work left no acknowledged Prompt record now ends with accounting limitations rather than plain completion. Adds a deterministic witness for both native readers and the comparison labels. Co-Authored-By: Claude Opus 5.5 --- examples/agent-usage/calibrate.ts | 4 +- examples/agent-usage/evidence.ts | 15 +- .../cli/tests/agent-usage-calibration.test.ts | 143 ++++++++++++++++++ 3 files changed, 159 insertions(+), 3 deletions(-) diff --git a/examples/agent-usage/calibrate.ts b/examples/agent-usage/calibrate.ts index e86d301f9..9ea6ccf03 100644 --- a/examples/agent-usage/calibrate.ts +++ b/examples/agent-usage/calibrate.ts @@ -527,7 +527,9 @@ function verdict(report: Report): "completed" | "limited" | "incomplete" { return "incomplete"; } const accounting = report.prompts.flatMap((prompt) => Object.values(prompt.checks)); - const limited = [...required, ...accounting].some((item) => item.outcome !== "verified"); + // Work that ran and left no acknowledged record is consumption nothing accounts for. + const unaccounted = report.submittedPrompts > report.acknowledgedPrompts; + const limited = unaccounted || [...required, ...accounting].some((item) => item.outcome !== "verified"); return limited || report.prompts.some((prompt) => prompt.usage?.coverage.outerPrompt !== "verified") ? "limited" : "completed"; } diff --git a/examples/agent-usage/evidence.ts b/examples/agent-usage/evidence.ts index b0ac0c8e9..74808900c 100644 --- a/examples/agent-usage/evidence.ts +++ b/examples/agent-usage/evidence.ts @@ -290,6 +290,8 @@ export function* claudeNative(home: string, sessionId: string): Operation; end: Record; last?: Record; @@ -343,7 +345,13 @@ export function* codexNative(home: string, threadId: string): Operation { ]), ); }); + + it("UC4: native accounts separate a whole Prompt from its final request, and find subagents", function* () { + const home = yield* useRoot(); + const codex = join(home, ".codex", "sessions", "2026", "11", "07"); + yield* ensureDir(codex); + const total = (input: number, output: number) => ({ + input_tokens: input, + cached_input_tokens: 0, + output_tokens: output, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + const rollout = [ + { type: "event_msg", payload: { type: "task_started" } }, + { type: "response_item", payload: { type: "function_call", name: "spawn_agent" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: total(100, 10), last_token_usage: total(100, 10) }, + }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: total(250, 15), last_token_usage: total(150, 5) }, + }, + }, + { type: "event_msg", payload: { type: "task_started" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: total(400, 20), last_token_usage: total(150, 5) }, + }, + }, + ]; + yield* writeTextFile( + join(codex, "rollout-2026-11-07T00-00-00-thread-1.jsonl"), + rollout.map((line) => JSON.stringify(line)).join("\n"), + ); + const read = yield* codexNative(home, "thread-1"); + expect( + read?.prompts.map(({ calls, toolCalls, subagentCalls, outer }) => ({ + calls, + toolCalls, + subagentCalls, + total: outer?.total_tokens, + })), + ).toEqual([ + { calls: 2, toolCalls: 1, subagentCalls: 1, total: 265 }, + { calls: 1, toolCalls: 0, subagentCalls: 0, total: 155 }, + ]); + const lastOnly = usageWith({ + inputTokens: 150, + outputTokens: 5, + cachedReadTokens: 0, + thoughtTokens: 0, + totalTokens: 155, + }); + expect(nativeComparison("codex", lastOnly, read?.prompts[0])).toEqual({ + outcome: "limited", + reason: "final-request-only", + }); + expect(nativeComparison("codex", lastOnly, read?.prompts[1])).toEqual({ + outcome: "verified", + reason: "matches-native-outer-total", + }); + expect(nativeComparison("codex", undefined, read?.prompts[0]).outcome).toBe("unavailable"); + + const claude = join(home, ".claude", "projects", "-fixture"); + yield* ensureDir(claude); + const usage = { + input_tokens: 3, + output_tokens: 2, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }; + const transcript = [ + { type: "user", message: { content: "read the files" } }, + { type: "user", isMeta: true, message: { content: "a reminder" } }, + { type: "assistant", message: { id: "m1", usage, content: [{ type: "tool_use" }] } }, + { type: "assistant", message: { id: "m1", usage, content: [{ type: "tool_use" }] } }, + { type: "user", message: { content: [{ type: "tool_result" }] } }, + { type: "assistant", isSidechain: true, message: { id: "s1", usage, content: [] } }, + { type: "assistant", message: { id: "m2", usage, content: [{ type: "text" }] } }, + ]; + yield* writeTextFile( + join(claude, "session-1.jsonl"), + transcript.map((line) => JSON.stringify(line)).join("\n"), + ); + const main = yield* claudeNative(home, "session-1"); + expect(main?.prompts).toEqual([ + { + calls: 2, + toolCalls: 2, + outer: { + input_tokens: 6, + output_tokens: 4, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + subagent: { + input_tokens: 3, + output_tokens: 2, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + subagentCalls: 1, + }, + ]); + expect( + nativeComparison( + "claude", + usageWith({ + inputTokens: 6, + outputTokens: 4, + cachedReadTokens: 0, + cachedWriteTokens: 0, + totalTokens: 10, + }), + main?.prompts[0], + ), + ).toEqual({ outcome: "verified", reason: "matches-native-outer-total-multicall" }); + }); }); + +function usageWith(tokens: Record): PromptUsage { + return { + provenance: { host: "compiled", xmd: {}, runtime: {}, adapter: {} }, + readings: [ + { source: "response", scope: "last-request", kind: "tokens", counter: "snapshot", tokens }, + ], + coverage: { finalReport: "received", outerPrompt: "partial", limitations: [] }, + }; +} From b05fe0c74439fb1083c869bcd8d700f108077b8e Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 06:35:25 -0500 Subject: [PATCH 03/10] test(cli): an ordinary run retains Prompt usage only in its selected journal Co-Authored-By: Claude Opus 5.5 --- .../cli/tests/agent-usage-calibration.test.ts | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/cli/tests/agent-usage-calibration.test.ts b/packages/cli/tests/agent-usage-calibration.test.ts index 7979e79a9..b5f50608c 100644 --- a/packages/cli/tests/agent-usage-calibration.test.ts +++ b/packages/cli/tests/agent-usage-calibration.test.ts @@ -27,7 +27,7 @@ import { nativeComparison, } from "../../../examples/agent-usage/evidence.ts"; import type { PromptUsage } from "@executablemd/core"; -import { writeTextFile } from "@effectionx/fs"; +import { readdir as readdirOf, writeTextFile } from "@effectionx/fs"; const CHECKOUT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); const DRIVER = join(CHECKOUT, "examples/agent-usage/calibrate.ts"); @@ -202,6 +202,44 @@ describe("Tier UC — usage calibration driver", () => { ); }); + it("UC5: an ordinary run retains usage only in the journal it was asked to keep", function* () { + const root = yield* useRoot(); + const document = join(root, "doc.md"); + yield* writeTextFile(document, 'measure me\n'); + const cli = [ + process.execPath, + "run", + "--allow-all", + "--frozen", + join(CHECKOUT, "packages/cli/src/deno.ts"), + ]; + const runOnce = (extra: string[]) => + scoped(function* () { + yield* useQuietProcessOutput(); + return yield* exec({ + command: [...cli, "run", document, ...extra], + cwd: root, + env: { ...inherited(), CLAUDE_CODE_EXECUTABLE: FAKE_CLAUDE }, + retain: true, + timeout: 120_000, + }); + }); + const transient = yield* runOnce([]); + expect(transient.exitCode).toBe(0); + expect((yield* readdirOf(root)).sort()).toEqual(["doc.md"]); + const journal = join(root, "run.jsonl"); + const retained = yield* runOnce(["--journal", journal]); + expect(retained.exitCode).toBe(0); + expect((yield* readdirOf(root)).sort()).toEqual(["doc.md", "run.jsonl"]); + const records = (yield* readTextFile(journal)) + .split("\n") + .filter((line) => line.includes('"agent_prompt"')) + .map((line) => JSON.parse(line).result.value); + expect(records.length).toBe(1); + expect(records[0].usage.provenance.host).toBe("deno-source"); + expect(records[0].usage.provenance.adapter.version).toBe("0.70.0"); + }); + it("UC4: native accounts separate a whole Prompt from its final request, and find subagents", function* () { const home = yield* useRoot(); const codex = join(home, ".codex", "sessions", "2026", "11", "07"); From ef2a54c8d0f54a634d41bfc0ea5c3e73cead0663 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 06:38:06 -0500 Subject: [PATCH 04/10] test(acp): witness Claude's main-loop response usage and imputed quota zero Co-Authored-By: Claude Opus 5.5 --- packages/acp/tests/adapter-protocol.test.ts | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/acp/tests/adapter-protocol.test.ts b/packages/acp/tests/adapter-protocol.test.ts index 876f65701..6db3d9d22 100644 --- a/packages/acp/tests/adapter-protocol.test.ts +++ b/packages/acp/tests/adapter-protocol.test.ts @@ -300,6 +300,37 @@ describe("Tier EA — the embedded adapters' prompt-response metadata", () => { }); }); + it("EA12: Claude's response usage is its main loop, and quota fills what it never measured", function* () { + const adapter = yield* useAdapter("claude", CLAUDE_ENV); + yield* initialize(adapter); + const sessionId = yield* openSession(adapter); + + const response = yield* prompt(adapter, sessionId); + + // The fake native result reports one input and one output token and no + // cache use. The response view is that main-loop tally under ACP's names; + // the quota view mirrors it and supplies a reasoning zero the native source + // never reported, which is why a retained record says `claude-imputed-zero`. + expect(response["usage"]).toEqual({ + inputTokens: 1, + outputTokens: 1, + cachedReadTokens: 0, + cachedWriteTokens: 0, + totalTokens: 2, + }); + expect(metaOf(response, "quota")).toEqual({ + token_count: { + totalTokens: 2, + inputTokens: 1, + cachedInputTokens: 0, + cachedWriteTokens: 0, + outputTokens: 1, + reasoningOutputTokens: 0, + }, + model_usage: [], + }); + }); + it("EA4: interleaved Claude sessions each carry their own assistant message", function* () { const adapter = yield* useAdapter("claude", CLAUDE_ENV); yield* initialize(adapter); From d7db37118785ac976fd4afb286e92a2d473f4167 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 07:17:55 -0500 Subject: [PATCH 05/10] fix(agent-usage): producing-owner disclosure, truthful calibration and missing witnesses Usage selection now asks the producing execution's authenticated policy binding, with the protected inputs captured when that policy was selected, instead of the replaceable contextual inputs, and screens every supplied string including limitations (F1). The calibration driver separates reserved Prompts from observed submissions, classifies a Prompt that failed before the experiment as missing execution, identifies the requested write and the subagent request, starts the admission clock with the command, and starts no later invocation after one that did not exit successfully, reporting its cleanup as unproven rather than verified (F2 and the admission correction). Native accounts deduplicate tool calls by their own identity, count only reads of the run's fixture files, and are compared only with the native turn that carried the run's exact input (F3). Adds the frozen deterministic witnesses: controlled cancel, admission deadline with held cleanup, report-write failure, missing execution, Plan and compiled carriage, REPL entry publisher and Sidekick owner rows, Claude model overlap and Codex last-request adapter mapping, held replay, scanner opt-out protected input and the final whole-event gate (F4). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 + examples/agent-usage/README.md | 10 +- examples/agent-usage/calibrate.ts | 343 ++++++++++---- examples/agent-usage/evidence.ts | 305 ++++++++---- examples/agent-usage/report.ts | 17 +- packages/acp/tests/adapter-protocol.test.ts | 101 ++++ .../acp/tests/fixtures/fake-claude-cli.cjs | 10 +- .../tests/fixtures/fake-codex-app-server.cjs | 10 + .../cli/tests/agent-usage-calibration.test.ts | 444 +++++++++++++++++- .../repl/issue-883/sidekick-harness.ts | 4 + packages/cli/tests/prompt-usage-hosts.test.ts | 52 ++ packages/core/src/agent/usage.ts | 54 +-- packages/core/src/secrets/policy.ts | 55 ++- packages/core/tests/agent-usage.test.ts | 143 +++++- scripts/runtime-test-exclusions.ts | 6 + scripts/tests/prompt-usage-compiled.test.ts | 92 ++++ 16 files changed, 1416 insertions(+), 236 deletions(-) create mode 100644 scripts/tests/prompt-usage-compiled.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00aa7ea11..764bd5209 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,12 @@ jobs: - name: The Component the compiled binary carries run: deno test --allow-all --frozen scripts/tests/plan-component-compiled.test.ts + # The compiled entrypoint states its own Prompt usage host facts, a + # different assembly from the source CLI's, so its carriage is asserted + # against the binary this job just built. + - name: Prompt usage the compiled binary retains + run: deno test --allow-all --frozen scripts/tests/prompt-usage-compiled.test.ts + # The entry host a REPL entry's file work is retained by is assembled by # `compiled.ts`, which is a different entrypoint from the one the source # row runs. A binary built without it starts a REPL, performs the work and diff --git a/examples/agent-usage/README.md b/examples/agent-usage/README.md index 08454c886..c025049d1 100644 --- a/examples/agent-usage/README.md +++ b/examples/agent-usage/README.md @@ -33,7 +33,9 @@ overwrites earlier evidence. `--help` prints the grammar and starts nothing. | `subagent` | 1 | One native subagent where supported, or the specific gap. | Each Prompt is attempted once. Nothing is retried, repaired or restarted, and no -other scenario runs. Each Prompt has 60 seconds of work, and no invocation is +other scenario runs. A later invocation starts only after the one before it exited +successfully: a host that ended unsuccessfully may have failed its own cleanup, +so the resume continuation is never started after an unsuccessful seed. Each Prompt has 60 seconds of work, and no invocation is admitted after 300 seconds; either deadline interrupts the host's own work and then waits for it to finish. @@ -56,7 +58,11 @@ Calibration incomplete. Read /report.md before attempting another run. acknowledged Prompt's source-labelled readings and coverage, and one row per check — usage retained, request-to-message association, ACPX's saved copy, the independent native comparison, the scenario's own behavior, history -readback and joined teardown. Each row is `verified`, `limited`, `unavailable`, +readback and joined teardown. The counts separate the outer Prompts reserved +against the allowance from the submissions the evidence actually shows, which +are `unknown` when the evidence cannot account for every reservation. A native +comparison is made only with the native turn that carried this run's exact +input in the session the record names. Each row is `verified`, `limited`, `unavailable`, `failed` or `not_run`, with a fixed reason. A `limited` row is a measured bound, such as `final-request-only`: the reported diff --git a/examples/agent-usage/calibrate.ts b/examples/agent-usage/calibrate.ts index 9ea6ccf03..65b58ff5b 100644 --- a/examples/agent-usage/calibrate.ts +++ b/examples/agent-usage/calibrate.ts @@ -29,7 +29,7 @@ import { nativeComparison, readAcknowledged, } from "./evidence.ts"; -import type { Acknowledged, Check, NativeSession } from "./evidence.ts"; +import type { Acknowledged, Check, Fixture, NativeSession } from "./evidence.ts"; import { renderMarkdown } from "./report.ts"; import type { PromptReport, Report } from "./report.ts"; @@ -126,12 +126,16 @@ interface Measured { readonly nativeVersion?: string; } -function* measure(selection: Selection, cwd: string): Operation { +function* measure( + selection: Selection, + cwd: string, + environment: CalibrationEnvironment, +): Operation { const version = (yield* firstLine([...hostCommand(selection), "--version"], cwd))?.match(VERSION_TOKEN)?.[0]; // The executable the bundled adapter would bind, when the host names one. const bound = selection.agent === "claude" ? process.env.CLAUDE_CODE_EXECUTABLE : process.env.CODEX_PATH; - const native = (yield* firstLine([bound ?? selection.agent, "--version"], cwd))?.match(VERSION_TOKEN)?.[0]; + const native = (yield* firstLine([environment.native ?? bound ?? selection.agent, "--version"], cwd))?.match(VERSION_TOKEN)?.[0]; if (selection.host === "compiled" && selection.xmd !== undefined) { return { ...(version === undefined ? {} : { xmdVersion: version }), @@ -201,12 +205,45 @@ function* readyInAcpx(acpxRoot: string, fixture: string, since: number): Operati return false; } +/** + * The clocks and native home one calibration runs under. + * + * The approved values are the defaults; a deterministic test supplies shorter + * clocks and a private home through this same value, so no command-line knob + * exists to change what an operator's run is held to. + */ +export interface CalibrationEnvironment { + /** Work allowed per outer Prompt. */ + readonly promptMs: number; + /** No invocation is admitted after this long from the start of the command. */ + readonly admissionMs: number; + /** Extra work time one invocation gets beyond its Prompts' own. */ + readonly slackMs: number; + /** How often readiness and deadlines are checked while a child runs. */ + readonly pollMs: number; + /** Where ACPX and the native agents keep their own state. */ + readonly home: string; + /** The native executable whose version is measured, when not the host's bound one. */ + readonly native?: string; +} + +function approvedEnvironment(): CalibrationEnvironment { + return { + promptMs: PROMPT_SECONDS * 1000, + admissionMs: ADMISSION_SECONDS * 1000, + slackMs: 30_000, + pollMs: 250, + home: homedir(), + }; +} + /** * Run one child host invocation to its joined end. * * A deadline or readiness marker sends the child an interrupt, which is the * host's own structured cancellation, and then joins it: nothing is detached - * or killed to make a report timely. + * or killed to make a report timely, and joining waits as long as the child's + * own cleanup takes. */ function* invoke( selection: Selection, @@ -214,6 +251,7 @@ function* invoke( cwd: string, deadlineAt: number, stderrPath: string, + environment: CalibrationEnvironment, ): Operation { const argv = [ ...hostCommand(selection), @@ -241,19 +279,22 @@ function* invoke( }); let cancelledAfterReady = false; let deadline = false; - const workDeadline = Math.min(deadlineAt, started + invocation.prompts * PROMPT_SECONDS * 1000 + 30_000); + const workDeadline = Math.min( + deadlineAt, + started + invocation.prompts * environment.promptMs + environment.slackMs, + ); const status = yield* race([ child.join(), (function* (): Operation { while (true) { - yield* sleep(250); - if (invocation.cancelOn !== undefined && !cancelledAfterReady) { - if (yield* readyInAcpx(join(homedir(), ".acpx"), invocation.cancelOn, started)) { + yield* sleep(environment.pollMs); + if (invocation.cancelOn !== undefined && !cancelledAfterReady && !deadline) { + if (yield* readyInAcpx(join(environment.home, ".acpx"), invocation.cancelOn, started)) { cancelledAfterReady = true; process.kill(child.pid, "SIGINT"); } } - if (!deadline && Date.now() > workDeadline) { + if (!deadline && !cancelledAfterReady && Date.now() > workDeadline) { deadline = true; process.kill(child.pid, "SIGINT"); } @@ -265,21 +306,40 @@ function* invoke( } return { started: true, - ...(status.code === undefined ? {} : { code: status.code }), - ...(status.signal === undefined ? {} : { signal: status.signal }), + // The join reports an absent code or signal as null. + ...(typeof status.code === "number" ? { code: status.code } : {}), + ...(typeof status.signal === "string" ? { signal: status.signal } : {}), ...(invocation.cancelOn === undefined ? {} : { cancelledAfterReady }), ...(deadline ? { deadline } : {}), }; }); } -function* nativeFor(agent: Selection["agent"], record: Acknowledged | undefined): Operation { +/** + * Whether a joined child's XMD owner is known to have finished cleanly. + * + * Only a zero exit says so: a teardown failure reaches the command line as a + * nonzero exit, exactly as an ordinary Prompt failure does, so a process that + * merely closed proves it joined and nothing about its cleanup. + */ +function succeeded(joined: Joined): boolean { + return joined.code === 0 && joined.signal === undefined; +} + +function* nativeFor( + agent: Selection["agent"], + record: Acknowledged | undefined, + environment: CalibrationEnvironment, + fixture: Fixture, +): Operation { const id = record?.record.agentSessionId; if (id === undefined) { return undefined; } try { - return agent === "claude" ? yield* claudeNative(homedir(), id) : yield* codexNative(homedir(), id); + return agent === "claude" + ? yield* claudeNative(environment.home, id, fixture) + : yield* codexNative(environment.home, id, fixture); } catch { return undefined; } @@ -293,13 +353,30 @@ function shareable(usage: PromptUsage | undefined): PromptReport["usage"] { return { readings: usage.readings.map((reading) => { const { model, ...rest } = reading; - return model !== undefined && /^[A-Za-z0-9._:\-[\]]{1,80}$/u.test(model) ? reading : rest; + return model !== undefined && MODEL.test(model) ? reading : rest; }), coverage: usage.coverage, }; } -function* runScenario(selection: Selection, output: string): Operation { +const MODEL = /^[A-Za-z0-9._:\-[\]]{1,80}$/u; + +/** What one scenario actually did, beside the record the report is built from. */ +interface Observed { + readonly acknowledged: Acknowledged[]; + readonly joins: Joined[]; + readonly planned: number; + readonly stopped?: "admission-deadline" | "unsuccessful-owner"; + /** The native session each acknowledged record names, by record index. */ + readonly natives: (NativeSession | undefined)[]; +} + +function* runScenario( + selection: Selection, + output: string, + start: number, + environment: CalibrationEnvironment, +): Operation { const fixture = join(output, "fixture"); const support = join(output, "support"); yield* ensureDir(fixture); @@ -331,42 +408,60 @@ function* runScenario(selection: Selection, output: string): Operation { return [{ document: join(SCENARIOS, "subagent.md"), props: { ...base, first: fileA }, journal: journal("subagent"), prompts: 1 }]; } })(); - const submissions = plan.reduce((sum, invocation) => sum + invocation.prompts, 0); - if (submissions > ALLOWANCE[selection.scenario]) { + const planned = plan.reduce((sum, invocation) => sum + invocation.prompts, 0); + if (planned > ALLOWANCE[selection.scenario]) { throw new Error("the scenario plans more outer Prompts than its allowance"); } - const start = Date.now(); - const admissionAt = start + ADMISSION_SECONDS * 1000; - const measured = yield* measure(selection, fixture); + const admissionAt = start + environment.admissionMs; + const measured = yield* measure(selection, fixture, environment); const joins: Joined[] = []; - let admitted = 0; - let stopped: string | undefined; + let reserved = 0; + let stopped: Observed["stopped"]; for (const [index, invocation] of plan.entries()) { if (Date.now() > admissionAt) { stopped = "admission-deadline"; break; } - admitted += invocation.prompts; - const joined = yield* invoke(selection, invocation, fixture, admissionAt, join(support, `child-${index}-stderr.txt`)); - joins.push(joined); - if (joined.code === undefined && joined.signal !== undefined && joined.signal !== "SIGINT") { - stopped = "cleanup-failed"; + // A later invocation depends on the one before it having finished + // cleanly. A child that did not exit zero either failed its work or its + // owner's cleanup, and nothing here can tell which, so nothing more starts. + const previous = joins.at(-1); + if (previous !== undefined && !succeeded(previous)) { + stopped = "unsuccessful-owner"; break; } + reserved += invocation.prompts; + joins.push( + yield* invoke(selection, invocation, fixture, admissionAt, join(support, `child-${index}-stderr.txt`), environment), + ); + } + if (stopped === "unsuccessful-owner") { + console.error(Refusal.cleanup); } const acknowledged: Acknowledged[] = []; for (const invocation of plan) { acknowledged.push(...((yield* readAcknowledged(invocation.journal)) ?? [])); } - const firstNative = yield* nativeFor(selection.agent, acknowledged[0]); + for (const entry of acknowledged) { + if (entry.record.status !== "completed" && entry.record.error?.message !== undefined) { + // The host's own refusal, as the operator would have seen it. + console.error(`Agent prompt ${entry.record.status}: ${entry.record.error.message}`); + } + } + const files: Fixture = { files: [fileA, fileB, deniedFile] }; + const natives: (NativeSession | undefined)[] = []; + for (const entry of acknowledged) { + natives.push(yield* nativeFor(selection.agent, entry, environment, files)); + } const prompts: PromptReport[] = []; const supportRecords: unknown[] = []; for (const [index, entry] of acknowledged.entries()) { const usage = entry.record.usage; - const native = selection.scenario === "resume" ? (yield* nativeFor(selection.agent, entry))?.prompts.at(index) : firstNative?.prompts[index]; - const acpx = yield* acpxAgreement(join(homedir(), ".acpx"), usage); + const native = natives[index]; + const associated = native?.associate(entry.input); + const acpx = yield* acpxAgreement(join(environment.home, ".acpx"), usage); prompts.push({ status: entry.record.status, usage: shareable(usage), @@ -374,33 +469,43 @@ function* runScenario(selection: Selection, output: string): Operation { usage: usage === undefined ? check("unavailable", "usage-unavailable") : check("verified", "usage-retained"), association: acpx.association, persistedAgreement: acpx.agreement, - independentNative: nativeComparison(selection.agent, usage, native), + independentNative: nativeComparison(selection.agent, usage, native, entry.input), }, - native: native === undefined ? undefined : { calls: native.calls, toolCalls: native.toolCalls, subagentCalls: native.subagentCalls }, + native: + associated === undefined + ? undefined + : { + calls: associated.calls, + toolCalls: associated.toolCalls, + fixtureReads: associated.fixtureReads, + subagentCalls: associated.subagentCalls, + }, }); supportRecords.push({ name: entry.name, sessionKey: entry.record.sessionKey, agentSessionId: entry.record.agentSessionId, usage, - native: native ?? null, - nativeSource: firstNative?.source ?? null, + native: associated ?? null, + nativeAssociation: native === undefined ? "native-source-unavailable" : associated === undefined ? "native-association-unavailable" : "delivered-input-in-one-native-turn", + nativeSource: native?.source ?? null, }); } yield* writeTextFile(join(support, "association.json"), `${JSON.stringify({ measured, prompts: supportRecords }, null, 2)}\n`); - const scenario = scenarioChecks(selection, acknowledged, joins, firstNative, markers, yield* exists(deniedFile)); - const teardown = stopped === "cleanup-failed" - ? check("failed", "owned-cleanup-failed") - : joins.length === plan.length - ? check("verified", "owner-joined") - : check("not_run", stopped ?? "not-admitted"); + const observed: Observed = { acknowledged, joins, planned, natives, ...(stopped === undefined ? {} : { stopped }) }; + const scenario = scenarioChecks(selection, observed, markers, yield* exists(deniedFile), deniedFile); + const witnessed = + acknowledged.filter((entry) => entry.record.usage?.providerMessageId !== undefined).length + + (selection.scenario === "cancel" && joins[0]?.cancelledAfterReady === true && acknowledged.length === 0 ? 1 : 0); return { agent: selection.agent, host: selection.host, scenario: selection.scenario, allowance: ALLOWANCE[selection.scenario], - submittedPrompts: admitted, + reservedPrompts: reserved, + observedSubmissions: witnessed === reserved ? witnessed : "unknown", + observedAtLeast: witnessed, acknowledgedPrompts: acknowledged.length, provenance: { xmd: { @@ -415,24 +520,61 @@ function* runScenario(selection: Selection, output: string): Operation { ...(acknowledged[0]?.record.usage?.provenance.native ?? {}), ...(measured.nativeVersion === undefined ? {} : { observedVersion: measured.nativeVersion }), }, - models: [...new Set(acknowledged.flatMap((entry) => entry.record.usage?.readings.flatMap((reading) => reading.model !== undefined && /^[A-Za-z0-9._:\-[\]]{1,80}$/u.test(reading.model) ? [reading.model] : []) ?? []))], + models: [...new Set(acknowledged.flatMap((entry) => entry.record.usage?.readings.flatMap((reading) => reading.model !== undefined && MODEL.test(reading.model) ? [reading.model] : []) ?? []))], }, prompts, scenarioChecks: scenario, readback: acknowledged.length > 0 ? check("verified", "canonical-journal-read-back") : check("unavailable", "no-acknowledged-history"), - teardown, + teardown: teardownCheck(selection, observed, plan.length), }; } +/** Whether this scenario's own expected path ends its one child unsuccessfully. */ +function expectsUnsuccessfulChild(selection: Selection): boolean { + return selection.scenario === "failure" || selection.scenario === "cancel"; +} + +function teardownCheck(selection: Selection, observed: Observed, planned: number): Check { + if (observed.stopped === "unsuccessful-owner") { + return check("failed", "unsuccessful-owner-stopped-admission"); + } + if (observed.stopped === "admission-deadline") { + return check("not_run", "admission-deadline"); + } + if (observed.joins.length !== planned) { + return check("not_run", "not-admitted"); + } + if (observed.joins.every(succeeded)) { + return check("verified", "owner-cleanup-succeeded"); + } + // An expected Agent failure or cancellation exits nonzero like a cleanup + // failure would: the owner joined, and its cleanup stays unproven. + return expectsUnsuccessfulChild(selection) + ? check("limited", "joined-cleanup-unproven") + : check("unavailable", "joined-cleanup-unproven"); +} + +function failedBeforeWork(entry: Acknowledged | undefined): boolean { + return entry !== undefined && entry.record.status !== "completed" && (entry.record.permissions ?? []).length === 0; +} + function scenarioChecks( selection: Selection, - acknowledged: Acknowledged[], - joins: Joined[], - native: NativeSession | undefined, + observed: Observed, markers: { a: string; b: string; r: string }, deniedWritten: boolean, + deniedFile: string, ): Record { + const { acknowledged, joins, natives } = observed; const completed = acknowledged.filter((entry) => entry.record.status === "completed"); + const associated = (index: number) => { + const entry = acknowledged[index]; + return entry === undefined ? undefined : natives[index]?.associate(entry.input); + }; + const nativeGap = (index: number) => + natives[index] === undefined + ? check("unavailable", "native-source-unavailable") + : check("unavailable", "native-association-unavailable"); switch (selection.scenario) { case "success": { const sameSession = @@ -440,16 +582,18 @@ function scenarioChecks( acknowledged[0].record.sessionKey === acknowledged[1].record.sessionKey && acknowledged[0].record.agentSessionId !== undefined && acknowledged[0].record.agentSessionId === acknowledged[1].record.agentSessionId; - const reads = native?.prompts[0]?.toolCalls; + const first = associated(0); return { twoPromptsOneSession: completed.length === 2 && sameSession ? check("verified", "two-acknowledged-prompts-one-session") - : check("failed", "two-prompts-not-observed"), - multicall: reads === undefined - ? check("unavailable", "native-tool-evidence-unavailable") - : reads >= 2 - ? check("verified", "two-or-more-native-tool-calls") - : check("failed", "fewer-than-two-native-tool-calls"), + : acknowledged.some((entry) => entry.record.status !== "completed") + ? check("failed", "prompts-failed") + : check("failed", "two-prompts-not-observed"), + multicall: first === undefined + ? nativeGap(0) + : first.fixtureReads >= 2 && first.fixtureFilesRead === 2 + ? check("verified", "two-separate-fixture-reads") + : check("failed", "two-separate-fixture-reads-not-observed"), recall: acknowledged[1]?.record.text.includes(markers.a) && acknowledged[1]?.record.text.includes(markers.b) ? check("verified", "markers-recalled") : check("limited", "markers-not-recalled"), @@ -463,7 +607,10 @@ function scenarioChecks( seed.record.agentSessionId !== undefined && continuation.record.agentSessionId === seed.record.agentSessionId; return { - separateInvocations: joins.length === 2 ? check("verified", "two-joined-invocations") : check("failed", "separate-invocation-not-observed"), + seed: seed?.record.status === "completed" && joins[0] !== undefined && succeeded(joins[0]) + ? check("verified", "seed-completed") + : check("failed", "seed-unsuccessful"), + separateInvocations: joins.length === 2 ? check("verified", "two-joined-invocations") : check("failed", "continuation-not-admitted"), exactTarget: continued && continuation?.record.status === "completed" ? check("verified", "exact-retained-target-continued") : continuation !== undefined && continuation.record.status !== "completed" @@ -473,18 +620,33 @@ function scenarioChecks( }; } case "failure": { - const refused = acknowledged[0]?.record.permissions?.some((permission) => - permission.outcome.outcome === "cancelled" || - permission.options.some((option) => option.optionId === (permission.outcome.outcome === "selected" ? permission.outcome.optionId : "") && option.kind.startsWith("reject")), + const entry = acknowledged[0]; + const permissions = entry?.record.permissions ?? []; + // The requested write, by what the host was asked to permit: an edit, + // or a request that names this run's private denied file. Another tool + // refused on the way is not the experiment. + const requested = permissions.filter( + (permission) => permission.kind === "edit" || (permission.title ?? "").includes(deniedFile), ); + const refused = requested.some((permission) => { + const outcome = permission.outcome; + if (outcome.outcome === "cancelled") { + return true; + } + return permission.options.some((option) => option.optionId === outcome.optionId && option.kind.startsWith("reject")); + }); return { - attemptedWriteRefused: refused === true - ? check("verified", "host-refused-attempted-native-write") - : check("failed", "native-write-not-attempted"), + attemptedWriteRefused: refused + ? check("verified", "host-refused-requested-native-write") + : entry === undefined + ? check("failed", "no-acknowledged-record") + : failedBeforeWork(entry) + ? check("failed", "prompt-failed-before-write") + : check("failed", "requested-write-not-attempted"), nothingWritten: deniedWritten ? check("failed", "denied-file-written") : check("verified", "denied-file-absent"), - acknowledgement: acknowledged.length === 1 - ? check("verified", `acknowledged-${acknowledged[0].record.status}`) - : check("limited", "no-acknowledged-record"), + acknowledgement: entry === undefined + ? check("limited", "no-acknowledged-record") + : check("verified", refused && entry.record.status === "completed" ? "acknowledged-completed-after-refusal" : `acknowledged-${entry.record.status}`), }; } case "cancel": { @@ -499,16 +661,29 @@ function scenarioChecks( }; } case "subagent": { - const observed = native?.prompts[0]?.subagentCalls; + const entry = acknowledged[0]; + if (entry === undefined || entry.record.status !== "completed") { + // Unavailability is a finding only about an experiment that ran. + return { + nativeSubagent: check("failed", "experiment-not-executed"), + inclusion: check("not_run", "experiment-not-executed"), + }; + } + const native = associated(0); + if (native === undefined) { + return { nativeSubagent: nativeGap(0), inclusion: check("not_run", "no-subagent-work") }; + } + if (native.subagentCalls > 0) { + return { + nativeSubagent: check("verified", "native-subagent-requested"), + inclusion: check("limited", "subagent-inclusion-unverified"), + }; + } return { - nativeSubagent: observed === undefined - ? check("unavailable", "native-source-unavailable") - : observed > 0 - ? check("verified", "native-subagent-work-observed") - : check("limited", "native-subagent-not-observed"), - inclusion: observed !== undefined && observed > 0 - ? check("limited", "subagent-inclusion-unverified") - : check("not_run", "no-subagent-work"), + nativeSubagent: /\bunavailable\b/iu.test(entry.record.text) + ? check("limited", "subagents-reported-unavailable") + : check("limited", "native-subagent-not-observed"), + inclusion: check("not_run", "no-subagent-work"), }; } } @@ -517,9 +692,13 @@ function scenarioChecks( /** Every check a selected scenario requires, and whether any of them is missing. */ function verdict(report: Report): "completed" | "limited" | "incomplete" { const required = Object.values(report.scenarioChecks); - if (report.teardown.outcome !== "verified" || required.some((item) => item.outcome === "failed" || item.outcome === "not_run" && item.reason !== "no-subagent-work")) { + const teardown = report.teardown; + const joined = teardown.outcome === "verified" || (teardown.outcome === "limited" && teardown.reason === "joined-cleanup-unproven"); + if (!joined || required.some((item) => item.outcome === "failed" || (item.outcome === "not_run" && item.reason !== "no-subagent-work"))) { return "incomplete"; } + // Only a subagent experiment that actually ran may end with its native + // evidence unavailable; every other required witness must have occurred. if (required.some((item) => item.outcome === "unavailable") && report.scenario !== "subagent") { return "incomplete"; } @@ -528,8 +707,8 @@ function verdict(report: Report): "completed" | "limited" | "incomplete" { } const accounting = report.prompts.flatMap((prompt) => Object.values(prompt.checks)); // Work that ran and left no acknowledged record is consumption nothing accounts for. - const unaccounted = report.submittedPrompts > report.acknowledgedPrompts; - const limited = unaccounted || [...required, ...accounting].some((item) => item.outcome !== "verified"); + const unaccounted = report.reservedPrompts > report.acknowledgedPrompts; + const limited = unaccounted || teardown.outcome !== "verified" || [...required, ...accounting].some((item) => item.outcome !== "verified"); return limited || report.prompts.some((prompt) => prompt.usage?.coverage.outerPrompt !== "verified") ? "limited" : "completed"; } @@ -576,7 +755,12 @@ function* quietChildren(): Operation { ); } -export function* calibrate(args: readonly string[]): Operation { +export function* calibrate( + args: readonly string[], + environment: CalibrationEnvironment = approvedEnvironment(), +): Operation { + // The admission clock covers the whole command, preparation included. + const start = Date.now(); const parsed = parseSelection(args); if (parsed.kind === "help") { console.log(HELP); @@ -596,19 +780,18 @@ export function* calibrate(args: readonly string[]): Operation { console.error(Refusal.exists(output)); return 1; } - const report = yield* runScenario(selection, output); + const report = yield* runScenario(selection, output, start, environment); const outcome = verdict(report); try { yield* writeTextFile(join(output, "report.json"), `${JSON.stringify({ ...report, result: outcome }, null, 2)}\n`); yield* writeTextFile(join(output, "report.md"), renderMarkdown(report, outcome)); yield* writeIndex(output); } catch (error) { + // The paid work already happened and is not repeated; no report is + // claimed where none could be written. console.error(`Calibration report could not be written: ${error instanceof Error ? error.message : String(error)}`); return 1; } - if (report.teardown.outcome === "failed") { - console.error(Refusal.cleanup); - } console.log(COMPLETION[outcome](output)); return outcome === "incomplete" ? 1 : 0; } diff --git a/examples/agent-usage/evidence.ts b/examples/agent-usage/evidence.ts index 74808900c..8d015cb4a 100644 --- a/examples/agent-usage/evidence.ts +++ b/examples/agent-usage/evidence.ts @@ -1,6 +1,6 @@ import { exists, readdir, readTextFile } from "@effectionx/fs"; import type { Operation } from "effection"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { parseDurableEvent } from "@executablemd/durable-streams"; import { AGENT_PROMPT, parsePromptRecord } from "@executablemd/core/host"; import type { PromptRecord } from "@executablemd/core/host"; @@ -22,6 +22,8 @@ export function check(outcome: Outcome, reason: string): Check { /** One canonical Prompt the history acknowledged, read back without a provider. */ export interface Acknowledged { readonly name: string; + /** The input this occurrence retained: the text the turn delivered. */ + readonly input: string; readonly record: PromptRecord; } @@ -55,7 +57,8 @@ export function* readAcknowledged(journal: string): Operation; /** The final internal request's counters, where the source can say. */ readonly last?: Record; /** Counters native subagent work recorded separately. */ readonly subagent?: Record; + /** Native subagent requests and separately recorded subagent model calls. */ readonly subagentCalls: number; } +/** + * One native Prompt while it is read, with what association needs. + * + * `userText` is only ever compared, inside this process, with the canonical + * input this run itself delivered; it is never written anywhere. + */ +interface Reading { + calls: number; + readonly tools: Set; + readonly reads: Set; + readonly files: Set; + readonly subagents: Set; + subagentCalls: number; + outer: Record; + subagent: Record; + last?: Record; + start: Record; + end: Record; + userText: string; + readonly seen: Set; +} + +function reading(start: Record = {}): Reading { + return { + calls: 0, + tools: new Set(), + reads: new Set(), + files: new Set(), + subagents: new Set(), + subagentCalls: 0, + outer: {}, + subagent: {}, + start, + end: start, + userText: "", + seen: new Set(), + }; +} + /** A native session's Prompts, located by its confirmed native identity, and where they came from. */ export interface NativeSession { - readonly prompts: NativePrompt[]; + readonly prompts: readonly NativePrompt[]; /** The exact private source file, for the private support record only. */ readonly source: string; + /** + * The one native Prompt whose user turn carried exactly this delivered input. + * + * Exact or nothing: no Prompt, or more than one, is an association gap, never + * a choice by position. + */ + associate(input: string): NativePrompt | undefined; +} + +function session(source: string, readings: readonly Reading[], summary: (reading: Reading) => NativePrompt): NativeSession { + const prompts = readings.map(summary); + return { + source, + prompts, + associate(input) { + const wanted = input.trim(); + if (wanted.length === 0) { + return undefined; + } + const matches = readings.flatMap((entry, index) => + entry.userText.includes(wanted) ? [prompts[index]] : [], + ); + return matches.length === 1 ? matches[0] : undefined; + }, + }; } function add(into: Record, from: Record, keys: readonly string[]) { @@ -190,6 +262,9 @@ const CLAUDE_KEYS = [ "cache_creation_input_tokens", ] as const; +/** Claude Code's own names for delegating to a subagent. */ +const CLAUDE_SUBAGENT_TOOLS = new Set(["Task", "Agent"]); + function* findFile(root: string, accept: (name: string) => boolean, depth: number): Operation { if (!(yield* exists(root))) { return undefined; @@ -223,21 +298,54 @@ function lines(text: string): Record[] { }); } +function textOf(content: unknown): string { + if (typeof content === "string") { + return content; + } + if (!Array.isArray(content)) { + return ""; + } + return content + .map((block) => { + const text = asRecord(block)?.text; + return typeof text === "string" ? text : ""; + }) + .join("\n"); +} + +/** The fixture files a native read may target, resolved against its working directory. */ +export interface Fixture { + readonly files: readonly string[]; +} + +function fixtureFile(fixture: Fixture | undefined, path: unknown, cwd: unknown): string | undefined { + if (fixture === undefined || typeof path !== "string") { + return undefined; + } + const resolved = resolve(typeof cwd === "string" ? cwd : "/", path); + return fixture.files.find((file) => resolve(file) === resolved); +} + /** * Claude Code's own transcript for one native session. * * A user entry that is a typed prompt rather than a tool result starts a * Prompt. Assistant entries repeat one API message per content block, so usage - * counts once per message id. Sidechain entries and subagent transcripts are - * the native account of subagent work, kept apart from the main loop. + * counts once per message id and a tool call once per tool-use id. Sidechain + * entries are the native account of subagent work, kept apart from the main + * loop; a `Task`/`Agent` tool use is the main loop asking for one. */ -export function* claudeNative(home: string, sessionId: string): Operation { +export function* claudeNative( + home: string, + sessionId: string, + fixture?: Fixture, +): Operation { const projects = join(home, ".claude", "projects"); const file = yield* findFile(projects, (name) => name === `${sessionId}.jsonl`, 1); if (file === undefined) { return undefined; } - const prompts: ClaudePrompt[] = []; + const readings: Reading[] = []; for (const entry of lines(yield* readTextFile(file))) { const message = asRecord(entry.message); if (entry.type === "user" && entry.isSidechain !== true && entry.isMeta !== true) { @@ -248,20 +356,40 @@ export function* claudeNative(home: string, sessionId: string): Operation asRecord(block)?.type === "text") && !content.some((block) => asRecord(block)?.type === "tool_result")); if (typed) { - prompts.push({ calls: 0, toolCalls: 0, outer: {}, subagent: {}, seen: new Set(), subagentCalls: 0 }); + const next = reading(); + next.userText = textOf(content); + readings.push(next); } continue; } - const current = prompts.at(-1); + const current = readings.at(-1); if (entry.type !== "assistant" || current === undefined || message === undefined) { continue; } const id = typeof message.id === "string" ? message.id : undefined; - const usage = asRecord(message.usage); const content = Array.isArray(message.content) ? message.content : []; - current.toolCalls += entry.isSidechain === true - ? 0 - : content.filter((block) => asRecord(block)?.type === "tool_use").length; + if (entry.isSidechain !== true) { + for (const [index, raw] of content.entries()) { + const block = asRecord(raw); + if (block?.type !== "tool_use") { + continue; + } + const tool = typeof block.id === "string" ? block.id : `${id ?? "message"}#${index}`; + current.tools.add(tool); + const input = asRecord(block.input); + if (block.name === "Read") { + const target = fixtureFile(fixture, input?.file_path, "/"); + if (target !== undefined) { + current.reads.add(tool); + current.files.add(target); + } + } + if (typeof block.name === "string" && CLAUDE_SUBAGENT_TOOLS.has(block.name)) { + current.subagents.add(tool); + } + } + } + const usage = asRecord(message.usage); if (id === undefined || usage === undefined || current.seen.has(id)) { continue; } @@ -274,37 +402,15 @@ export function* claudeNative(home: string, sessionId: string): Operation ({ - calls, - toolCalls, - outer, - ...(subagentCalls > 0 ? { subagent } : {}), - subagentCalls, - })), - }; -} - -/** One Codex Prompt while its rollout is being read. */ -interface CodexPrompt { - calls: number; - toolCalls: number; - /** Native subagents this Prompt's thread spawned; their own threads hold their usage. */ - subagentCalls: number; - start: Record; - end: Record; - last?: Record; -} - -/** One Claude Prompt while its transcript is being read. */ -interface ClaudePrompt { - calls: number; - toolCalls: number; - outer: Record; - subagent: Record; - subagentCalls: number; - readonly seen: Set; + return session(file, readings, (entry) => ({ + calls: entry.calls, + toolCalls: entry.tools.size, + fixtureReads: entry.reads.size, + fixtureFilesRead: entry.files.size, + outer: entry.outer, + ...(entry.subagentCalls > 0 ? { subagent: entry.subagent } : {}), + subagentCalls: entry.subagents.size + entry.subagentCalls, + })); } const CODEX_KEYS = [ @@ -322,17 +428,20 @@ const CODEX_KEYS = [ * Each `token_count` event carries both the running thread total and the last * request's reading. A started task starts a Prompt; the difference between the * running totals at its end and at the previous Prompt's end is the outer - * Prompt's own total, and the last reading in it is its final request. + * Prompt's own total, and the last reading in it is its final request. Tool + * calls count once per call id; a completed command whose parsed form is a read + * of a fixture file is a fixture read; `spawn_agent` is a native subagent. */ -export function* codexNative(home: string, threadId: string): Operation { +export function* codexNative( + home: string, + threadId: string, + fixture?: Fixture, +): Operation { const sessions = join(home, ".codex", "sessions"); const file = yield* findFile(sessions, (name) => name.endsWith(`${threadId}.jsonl`), 3); if (file === undefined) { return undefined; } - let previousTotal: Record = {}; - let runningTotal: Record = {}; - const prompts: CodexPrompt[] = []; const entries = lines(yield* readTextFile(file)); // A turn starts where the rollout says a task started; older rollouts mark // only the user message. @@ -341,27 +450,45 @@ export function* codexNative(home: string, threadId: string): Operation = {}; + const readings: Reading[] = []; for (const entry of entries) { const payload = asRecord(entry.payload); if (entry.type === "event_msg" && payload?.type === boundary) { - previousTotal = { ...runningTotal }; - prompts.push({ - calls: 0, - toolCalls: 0, - subagentCalls: 0, - start: previousTotal, - end: previousTotal, - }); + readings.push(reading({ ...runningTotal })); continue; } - const current = prompts.at(-1); + const current = readings.at(-1); if (current === undefined) { continue; } - if (entry.type === "response_item" && typeof payload?.type === "string" && /call$/u.test(payload.type) && !/output$/u.test(payload.type)) { - current.toolCalls++; + if (entry.type === "response_item" && payload?.type === "message" && payload.role === "user") { + current.userText += `${textOf(payload.content)}\n`; + } + if ( + entry.type === "response_item" && + typeof payload?.type === "string" && + /call$/u.test(payload.type) && + !/output$/u.test(payload.type) + ) { + const call = typeof payload.call_id === "string" ? payload.call_id : `call#${current.tools.size}`; + current.tools.add(call); if (payload.name === "spawn_agent") { - current.subagentCalls++; + current.subagents.add(call); + } + } + if (entry.type === "event_msg" && payload?.type === "item_completed") { + const item = asRecord(payload.item); + if (item?.type === "CommandExecution" && Array.isArray(item.parsed_cmd)) { + const id = typeof item.id === "string" ? item.id : `command#${current.reads.size}`; + for (const parsed of item.parsed_cmd) { + const command = asRecord(parsed); + const target = command?.type === "read" ? fixtureFile(fixture, command.path, item.cwd) : undefined; + if (target !== undefined) { + current.reads.add(id); + current.files.add(target); + } + } } } if (entry.type === "event_msg" && payload?.type === "token_count") { @@ -377,32 +504,31 @@ export function* codexNative(home: string, threadId: string): Operation = {}; - add(reading, last, CODEX_KEYS); - current.last = reading; + const latest: Record = {}; + add(latest, last, CODEX_KEYS); + current.last = latest; } } current.end = { ...runningTotal }; } - return { - source: file, - prompts: prompts.map((prompt) => { - const outer: Record = {}; - for (const key of CODEX_KEYS) { - const end = prompt.end[key]; - if (end !== undefined) { - outer[key] = end - (prompt.start[key] ?? 0); - } + return session(file, readings, (entry) => { + const outer: Record = {}; + for (const key of CODEX_KEYS) { + const end = entry.end[key]; + if (end !== undefined) { + outer[key] = end - (entry.start[key] ?? 0); } - return { - calls: prompt.calls, - toolCalls: prompt.toolCalls, - outer, - ...(prompt.last === undefined ? {} : { last: prompt.last }), - subagentCalls: prompt.subagentCalls, - }; - }), - }; + } + return { + calls: entry.calls, + toolCalls: entry.tools.size, + fixtureReads: entry.reads.size, + fixtureFilesRead: entry.files.size, + outer, + ...(entry.last === undefined ? {} : { last: entry.last }), + subagentCalls: entry.subagents.size, + }; + }); } /** The canonical response view, spelled in a native source's own counter names. */ @@ -443,6 +569,10 @@ function sameOn(left: Record, right: Record | un /** * Compare one acknowledged response view with the native account of the same Prompt. * + * The native Prompt is the one whose user turn, in the session the canonical + * record names, carried the exact input this run delivered; anything else is + * an association gap rather than a comparison. + * * `limited` when the canonical value matches only the final internal request * of a Prompt that made more than one: a bound, not a total. A disagreement is * a documented limitation, never a reason to manufacture a total. @@ -450,11 +580,16 @@ function sameOn(left: Record, right: Record | un export function nativeComparison( agent: "claude" | "codex", usage: PromptUsage | undefined, - native: NativePrompt | undefined, + session: NativeSession | undefined, + input: string, ): Check { - if (native === undefined) { + if (session === undefined) { return check("unavailable", "native-source-unavailable"); } + const native = session.associate(input); + if (native === undefined) { + return check("unavailable", "native-association-unavailable"); + } const response = tokensOf(usage, "response"); if (response === undefined || response === null) { return check("unavailable", "no-reported-response"); diff --git a/examples/agent-usage/report.ts b/examples/agent-usage/report.ts index 5226015b8..856af1da0 100644 --- a/examples/agent-usage/report.ts +++ b/examples/agent-usage/report.ts @@ -14,7 +14,12 @@ export interface PromptReport { readonly persistedAgreement: Check; readonly independentNative: Check; }; - readonly native?: { readonly calls: number; readonly toolCalls: number; readonly subagentCalls: number }; + readonly native?: { + readonly calls: number; + readonly toolCalls: number; + readonly fixtureReads: number; + readonly subagentCalls: number; + }; } /** @@ -29,7 +34,11 @@ export interface Report { readonly host: "source" | "compiled"; readonly scenario: string; readonly allowance: number; - readonly submittedPrompts: number; + /** Outer Prompts reserved against the allowance by the invocations admitted. */ + readonly reservedPrompts: number; + /** Submissions the evidence shows, or unknown when it cannot account for every reservation. */ + readonly observedSubmissions: number | "unknown"; + readonly observedAtLeast: number; readonly acknowledgedPrompts: number; readonly provenance: { readonly xmd: { @@ -75,7 +84,7 @@ export function renderMarkdown(report: Report, result: string): string { const out: string[] = [ `# Usage calibration: ${report.agent} on ${report.host}, ${report.scenario}`, "", - `Result: **${result}**. Allowance ${report.allowance} outer Prompts; submitted ${report.submittedPrompts}; acknowledged ${report.acknowledgedPrompts}.`, + `Result: **${result}**. Allowance ${report.allowance} outer Prompts; reserved ${report.reservedPrompts}; observed submissions ${report.observedSubmissions === "unknown" ? `unknown (at least ${report.observedAtLeast})` : report.observedSubmissions}; acknowledged ${report.acknowledgedPrompts}.`, "", "## Builds", "", @@ -104,7 +113,7 @@ export function renderMarkdown(report: Report, result: string): string { out.push(""); if (prompt.native !== undefined) { out.push( - `Native account: ${prompt.native.calls} model calls, ${prompt.native.toolCalls} tool calls, ${prompt.native.subagentCalls} subagent calls.`, + `Native account: ${prompt.native.calls} model calls, ${prompt.native.toolCalls} tool calls, ${prompt.native.fixtureReads} fixture reads, ${prompt.native.subagentCalls} subagent calls.`, "", ); } diff --git a/packages/acp/tests/adapter-protocol.test.ts b/packages/acp/tests/adapter-protocol.test.ts index 6db3d9d22..db6e67eea 100644 --- a/packages/acp/tests/adapter-protocol.test.ts +++ b/packages/acp/tests/adapter-protocol.test.ts @@ -331,6 +331,107 @@ describe("Tier EA — the embedded adapters' prompt-response metadata", () => { }); }); + it("EA13: Claude keeps its broader per-model tally beside, not inside, its main-loop usage", function* () { + const adapter = yield* useAdapter("claude", { + ...CLAUDE_ENV, + FAKE_CLAUDE_RESULT_USAGE: JSON.stringify({ + input_tokens: 3, + output_tokens: 2, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 1, + }), + // The SDK's modelUsage also counts subagents and internal calls, so it + // is larger than the main loop's own result. + FAKE_CLAUDE_MODEL_USAGE: JSON.stringify({ + "fake-model": { + inputTokens: 7, + outputTokens: 5, + cacheReadInputTokens: 6, + cacheCreationInputTokens: 2, + }, + }), + }); + yield* initialize(adapter); + const sessionId = yield* openSession(adapter); + + const response = yield* prompt(adapter, sessionId); + + expect(response["usage"]).toEqual({ + inputTokens: 3, + outputTokens: 2, + cachedReadTokens: 4, + cachedWriteTokens: 1, + totalTokens: 10, + }); + expect(metaOf(response, "quota")).toEqual({ + token_count: { + totalTokens: 10, + inputTokens: 3, + cachedInputTokens: 4, + cachedWriteTokens: 1, + outputTokens: 2, + reasoningOutputTokens: 0, + }, + model_usage: [ + { + model: "fake-model", + token_count: { + totalTokens: 20, + inputTokens: 7, + cachedInputTokens: 6, + cachedWriteTokens: 2, + outputTokens: 5, + reasoningOutputTokens: 0, + }, + }, + ], + }); + }); + + it("EA14: Codex reports only its last internal request, net of cached input, with reasoning apart", function* () { + const reading = (input: number, cached: number, output: number, reasoning: number) => ({ + totalTokens: input + output, + inputTokens: input, + cachedInputTokens: cached, + outputTokens: output, + reasoningOutputTokens: reasoning, + }); + const adapter = yield* useAdapter("codex", { + ...CODEX_ENV, + // Two internal requests in one turn: the second's reading is the last, + // and the thread total covers both. + FAKE_CODEX_TOKEN_USAGE: JSON.stringify([ + { last: reading(100, 40, 10, 3), total: reading(100, 40, 10, 3), modelContextWindow: 1000 }, + { last: reading(150, 90, 5, 2), total: reading(250, 130, 15, 5), modelContextWindow: 1000 }, + ]), + }); + yield* initialize(adapter); + const sessionId = yield* openSession(adapter); + + const response = yield* prompt(adapter, sessionId); + + expect(response["usage"]).toEqual({ + totalTokens: 155, + inputTokens: 60, + cachedReadTokens: 90, + outputTokens: 5, + thoughtTokens: 2, + }); + const quota = metaOf(response, "quota"); + const lastCount = { + totalTokens: 155, + inputTokens: 60, + cachedInputTokens: 90, + outputTokens: 5, + reasoningOutputTokens: 2, + }; + expect(Reflect.get(Object(quota), "token_count")).toEqual(lastCount); + const rows = Reflect.get(Object(quota), "model_usage"); + expect( + Array.isArray(rows) ? rows.map((row) => Reflect.get(Object(row), "token_count")) : rows, + ).toEqual([lastCount]); + }); + it("EA4: interleaved Claude sessions each carry their own assistant message", function* () { const adapter = yield* useAdapter("claude", CLAUDE_ENV); yield* initialize(adapter); diff --git a/packages/acp/tests/fixtures/fake-claude-cli.cjs b/packages/acp/tests/fixtures/fake-claude-cli.cjs index d605eeaae..c9ff73584 100755 --- a/packages/acp/tests/fixtures/fake-claude-cli.cjs +++ b/packages/acp/tests/fixtures/fake-claude-cli.cjs @@ -247,8 +247,14 @@ process.stdin.on("data", (chunk) => { duration_api_ms: 1, num_turns: 1, total_cost_usd: 0, - usage: usage(), - modelUsage: {}, + // A case may script the main loop's own result usage and the broader + // per-model tally the SDK reports beside it. + usage: process.env.FAKE_CLAUDE_RESULT_USAGE + ? JSON.parse(process.env.FAKE_CLAUDE_RESULT_USAGE) + : usage(), + modelUsage: process.env.FAKE_CLAUDE_MODEL_USAGE + ? JSON.parse(process.env.FAKE_CLAUDE_MODEL_USAGE) + : {}, permission_denials: [], uuid: crypto.randomUUID(), session_id: SESSION, diff --git a/packages/acp/tests/fixtures/fake-codex-app-server.cjs b/packages/acp/tests/fixtures/fake-codex-app-server.cjs index 052468551..bd73a2ffa 100755 --- a/packages/acp/tests/fixtures/fake-codex-app-server.cjs +++ b/packages/acp/tests/fixtures/fake-codex-app-server.cjs @@ -158,6 +158,16 @@ function handle(message) { // Held briefly so two prompts can be in flight at once, which is what // makes the interleaved case an actual overlap. setTimeout(() => { + // The token readings a real App Server reports as a turn's internal + // requests finish, when a case scripts them: each carries the last + // request's reading and the thread's running total. + for (const tokenUsage of JSON.parse(process.env.FAKE_CODEX_TOKEN_USAGE ?? "[]")) { + send({ + jsonrpc: "2.0", + method: "thread/tokenUsage/updated", + params: { threadId, turnId, tokenUsage }, + }); + } send({ jsonrpc: "2.0", method: "turn/completed", diff --git a/packages/cli/tests/agent-usage-calibration.test.ts b/packages/cli/tests/agent-usage-calibration.test.ts index b5f50608c..38bc1fb5a 100644 --- a/packages/cli/tests/agent-usage-calibration.test.ts +++ b/packages/cli/tests/agent-usage-calibration.test.ts @@ -21,6 +21,10 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import process from "node:process"; import { HELP, parseSelection, Refusal } from "../../../examples/agent-usage/selection.ts"; +import { calibrate } from "../../../examples/agent-usage/calibrate.ts"; +import type { CalibrationEnvironment } from "../../../examples/agent-usage/calibrate.ts"; +import { chmod } from "node:fs/promises"; +import { until } from "effection"; import { claudeNative, codexNative, @@ -155,17 +159,22 @@ describe("Tier UC — usage calibration driver", () => { ]); const report = JSON.parse(yield* readTextFile(join(output, "report.json"))); expect(report.result).toBe("incomplete"); - expect([report.allowance, report.submittedPrompts, report.acknowledgedPrompts]).toEqual([ - 2, 2, 2, - ]); + // Both Prompts were reserved, and both are witnessed as submitted by the + // user message the runtime generated for each. + expect([ + report.allowance, + report.reservedPrompts, + report.observedSubmissions, + report.acknowledgedPrompts, + ]).toEqual([2, 2, 2, 2]); expect(report.scenarioChecks.twoPromptsOneSession.outcome).toBe("verified"); // A controlled agent writes no native transcript: the required multicall // witness is missing, which is incomplete calibration, not a pass. expect(report.scenarioChecks.multicall).toEqual({ outcome: "unavailable", - reason: "native-tool-evidence-unavailable", + reason: "native-source-unavailable", }); - expect(report.teardown.outcome).toBe("verified"); + expect(report.teardown).toEqual({ outcome: "verified", reason: "owner-cleanup-succeeded" }); for (const prompt of report.prompts) { expect(prompt.checks.usage.outcome).toBe("verified"); expect(prompt.checks.association).toEqual({ @@ -240,8 +249,53 @@ describe("Tier UC — usage calibration driver", () => { expect(records[0].usage.provenance.adapter.version).toBe("0.70.0"); }); - it("UC4: native accounts separate a whole Prompt from its final request, and find subagents", function* () { + it("UC11: Plan retains its writer's usage only in the journal it was asked to keep", function* () { + const root = yield* useRoot(); + const cli = [ + process.execPath, + "run", + "--allow-all", + "--frozen", + join(CHECKOUT, "packages/cli/src/deno.ts"), + ]; + const plan = (extra: string[]) => + scoped(function* () { + yield* useQuietProcessOutput(); + return yield* exec({ + command: [ + ...cli, + "plan", + "write a hello document", + "--default-agent", + "claude", + ...extra, + ], + cwd: root, + env: { ...inherited(), CLAUDE_CODE_EXECUTABLE: FAKE_CLAUDE }, + retain: true, + timeout: 180_000, + }); + }); + // The controlled writer cannot produce a valid program, so Plan ends + // unsuccessfully either way; what is asserted is where its turns went. + yield* plan([]); + expect(yield* readdirOf(root)).toEqual([]); + yield* plan(["--journal", join(root, "plan.jsonl")]); + expect(yield* readdirOf(root)).toEqual(["plan.jsonl"]); + const records = (yield* readTextFile(join(root, "plan.jsonl"))) + .split("\n") + .filter((line) => line.includes('"agent_prompt"')) + .map((line) => JSON.parse(line).result.value); + expect(records.length).toBeGreaterThan(0); + for (const record of records) { + expect(record.usage.provenance.host).toBe("deno-source"); + expect(record.usage.provenance.adapter.version).toBe("0.70.0"); + } + }); + + it("UC4: native accounts count distinct calls, separate a whole Prompt from its final request, and associate exactly", function* () { const home = yield* useRoot(); + const fixture = { files: ["/fixture/a.txt", "/fixture/b.txt"] }; const codex = join(home, ".codex", "sessions", "2026", "11", "07"); yield* ensureDir(codex); const total = (input: number, output: number) => ({ @@ -251,9 +305,33 @@ describe("Tier UC — usage calibration driver", () => { reasoning_output_tokens: 0, total_tokens: input + output, }); + const userTurn = (text: string) => ({ + type: "response_item", + payload: { type: "message", role: "user", content: [{ type: "input_text", text }] }, + }); + const command = (id: string, path: string) => ({ + type: "event_msg", + payload: { + type: "item_completed", + item: { + type: "CommandExecution", + id, + cwd: "/fixture", + parsed_cmd: [{ type: "read", path }], + }, + }, + }); const rollout = [ { type: "event_msg", payload: { type: "task_started" } }, - { type: "response_item", payload: { type: "function_call", name: "spawn_agent" } }, + userTurn("\nread both fixture files"), + { type: "response_item", payload: { type: "custom_tool_call", name: "exec", call_id: "c1" } }, + command("i1", "a.txt"), + { type: "response_item", payload: { type: "custom_tool_call", name: "exec", call_id: "c2" } }, + command("i2", "b.txt"), + { + type: "response_item", + payload: { type: "function_call", name: "spawn_agent", call_id: "c3" }, + }, { type: "event_msg", payload: { @@ -269,6 +347,7 @@ describe("Tier UC — usage calibration driver", () => { }, }, { type: "event_msg", payload: { type: "task_started" } }, + userTurn("repeat them"), { type: "event_msg", payload: { @@ -281,18 +360,42 @@ describe("Tier UC — usage calibration driver", () => { join(codex, "rollout-2026-11-07T00-00-00-thread-1.jsonl"), rollout.map((line) => JSON.stringify(line)).join("\n"), ); - const read = yield* codexNative(home, "thread-1"); + const read = yield* codexNative(home, "thread-1", fixture); expect( - read?.prompts.map(({ calls, toolCalls, subagentCalls, outer }) => ({ - calls, - toolCalls, - subagentCalls, - total: outer?.total_tokens, - })), + read?.prompts.map( + ({ calls, toolCalls, fixtureReads, fixtureFilesRead, subagentCalls, outer }) => ({ + calls, + toolCalls, + fixtureReads, + fixtureFilesRead, + subagentCalls, + total: outer?.total_tokens, + }), + ), ).toEqual([ - { calls: 2, toolCalls: 1, subagentCalls: 1, total: 265 }, - { calls: 1, toolCalls: 0, subagentCalls: 0, total: 155 }, + { + calls: 2, + toolCalls: 3, + fixtureReads: 2, + fixtureFilesRead: 2, + subagentCalls: 1, + total: 265, + }, + { + calls: 1, + toolCalls: 0, + fixtureReads: 0, + fixtureFilesRead: 0, + subagentCalls: 0, + total: 155, + }, ]); + // Association is by the delivered input in exactly one native turn of the + // confirmed session, never by position. + expect(read?.associate("read both fixture files")).toBe(read?.prompts[0]); + expect(read?.associate("repeat them")).toBe(read?.prompts[1]); + expect(read?.associate("a Prompt this session never saw")).toBe(undefined); + expect(read?.associate("re")).toBe(undefined); const lastOnly = usageWith({ inputTokens: 150, outputTokens: 5, @@ -300,15 +403,22 @@ describe("Tier UC — usage calibration driver", () => { thoughtTokens: 0, totalTokens: 155, }); - expect(nativeComparison("codex", lastOnly, read?.prompts[0])).toEqual({ + expect(nativeComparison("codex", lastOnly, read, "read both fixture files")).toEqual({ outcome: "limited", reason: "final-request-only", }); - expect(nativeComparison("codex", lastOnly, read?.prompts[1])).toEqual({ + expect(nativeComparison("codex", lastOnly, read, "repeat them")).toEqual({ outcome: "verified", reason: "matches-native-outer-total", }); - expect(nativeComparison("codex", undefined, read?.prompts[0]).outcome).toBe("unavailable"); + expect(nativeComparison("codex", lastOnly, read, "unknown input")).toEqual({ + outcome: "unavailable", + reason: "native-association-unavailable", + }); + expect(nativeComparison("codex", lastOnly, undefined, "repeat them")).toEqual({ + outcome: "unavailable", + reason: "native-source-unavailable", + }); const claude = join(home, ".claude", "projects", "-fixture"); yield* ensureDir(claude); @@ -318,11 +428,17 @@ describe("Tier UC — usage calibration driver", () => { cache_read_input_tokens: 0, cache_creation_input_tokens: 0, }; + const reads = [ + { type: "tool_use", id: "t1", name: "Read", input: { file_path: "/fixture/a.txt" } }, + { type: "tool_use", id: "t2", name: "Read", input: { file_path: "/fixture/b.txt" } }, + ]; const transcript = [ { type: "user", message: { content: "read the files" } }, { type: "user", isMeta: true, message: { content: "a reminder" } }, - { type: "assistant", message: { id: "m1", usage, content: [{ type: "tool_use" }] } }, - { type: "assistant", message: { id: "m1", usage, content: [{ type: "tool_use" }] } }, + // One API message, logged once per content block: one model call and + // two distinct tool calls, however many times the entry repeats. + { type: "assistant", message: { id: "m1", usage, content: reads } }, + { type: "assistant", message: { id: "m1", usage, content: reads } }, { type: "user", message: { content: [{ type: "tool_result" }] } }, { type: "assistant", isSidechain: true, message: { id: "s1", usage, content: [] } }, { type: "assistant", message: { id: "m2", usage, content: [{ type: "text" }] } }, @@ -331,11 +447,13 @@ describe("Tier UC — usage calibration driver", () => { join(claude, "session-1.jsonl"), transcript.map((line) => JSON.stringify(line)).join("\n"), ); - const main = yield* claudeNative(home, "session-1"); + const main = yield* claudeNative(home, "session-1", fixture); expect(main?.prompts).toEqual([ { calls: 2, toolCalls: 2, + fixtureReads: 2, + fixtureFilesRead: 2, outer: { input_tokens: 6, output_tokens: 4, @@ -361,10 +479,120 @@ describe("Tier UC — usage calibration driver", () => { cachedWriteTokens: 0, totalTokens: 10, }), - main?.prompts[0], + main, + "read the files", ), ).toEqual({ outcome: "verified", reason: "matches-native-outer-total-multicall" }); }); + + it("UC6: an unsuccessful seed stops its continuation; a successful seed admits it", function* () { + const failed = yield* controlled({ "resume-seed.md": { exit: 1 } }); + const stopped = yield* runControlled(failed, "resume"); + expect(stopped.status).toBe(1); + expect(stopped.stderr).toContain(Refusal.cleanup); + expect(yield* documents(failed)).toEqual(["resume-seed.md:started"]); + const report = JSON.parse(yield* readTextFile(join(stopped.output, "report.json"))); + expect(report.teardown).toEqual({ + outcome: "failed", + reason: "unsuccessful-owner-stopped-admission", + }); + expect(report.scenarioChecks.seed).toEqual({ outcome: "failed", reason: "seed-unsuccessful" }); + expect(report.scenarioChecks.separateInvocations.reason).toBe("continuation-not-admitted"); + expect([report.reservedPrompts, report.observedSubmissions]).toEqual([1, "unknown"]); + expect(report.result).toBe("incomplete"); + // Earlier evidence survives the stop. + expect(yield* exists(join(stopped.output, "support/child-0-stderr.txt"))).toBe(true); + + const control = yield* controlled({ "resume-seed.md": { exit: 0 } }); + const admitted = yield* runControlled(control, "resume"); + expect(yield* documents(control)).toEqual([ + "resume-seed.md:started", + "resume-continuation.md:started", + ]); + expect(admitted.stderr).not.toContain(Refusal.cleanup); + const controlReport = JSON.parse(yield* readTextFile(join(admitted.output, "report.json"))); + expect(controlReport.teardown).toEqual({ + outcome: "verified", + reason: "owner-cleanup-succeeded", + }); + expect(controlReport.reservedPrompts).toBe(2); + }); + + it("UC7: cancellation waits for the ready marker while work is active, then joins the owner", function* () { + const host = yield* controlled({ "cancel.md": { ready: true, hold: true, cleanupMs: 300 } }); + const ran = yield* runControlled(host, "cancel"); + expect(yield* documents(host)).toEqual([ + "cancel.md:started", + "cancel.md:interrupted", + "cancel.md:cleaned", + ]); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(report.scenarioChecks.cancelledWhileActive).toEqual({ + outcome: "verified", + reason: "cancelled-after-ready-marker", + }); + expect(report.scenarioChecks.acknowledgement.reason).toBe("no-acknowledged-record"); + expect(report.teardown).toEqual({ outcome: "limited", reason: "joined-cleanup-unproven" }); + expect([ + report.reservedPrompts, + report.observedSubmissions, + report.acknowledgedPrompts, + ]).toEqual([1, 1, 0]); + // Submitted work with no acknowledged record is an accounting gap. + expect([ran.status, report.result]).toEqual([0, "limited"]); + }); + + it("UC8: the admission deadline interrupts owned work, waits for its held cleanup and admits nothing later", function* () { + const host = yield* controlled({ "resume-seed.md": { hold: true, cleanupMs: 400 } }); + const ran = yield* runControlled(host, "resume", { admissionMs: 300, promptMs: 10_000 }); + expect(ran.status).toBe(1); + // Joined only after the held cleanup finished; the continuation never started. + expect(yield* documents(host)).toEqual([ + "resume-seed.md:started", + "resume-seed.md:interrupted", + "resume-seed.md:cleaned", + ]); + expect(ran.elapsedMs).toBeGreaterThanOrEqual(700); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(report.teardown).toEqual({ outcome: "not_run", reason: "admission-deadline" }); + expect(report.result).toBe("incomplete"); + }); + + it("UC9: a report that cannot be written is said so once, with no completion line and no retry", function* () { + const host = yield* controlled({ "failure.md": { exit: 1, blockReport: true } }); + const ran = yield* runControlled(host, "failure"); + expect(ran.status).toBe(1); + expect(ran.stderr).toContain("Calibration report could not be written"); + expect(ran.stdout).not.toContain("Calibration completed"); + expect(ran.stdout).not.toContain("Calibration incomplete"); + expect(yield* documents(host)).toEqual(["failure.md:started"]); + }); + + it("UC10: a Prompt that failed before the experiment is missing execution, not a limitation", function* () { + for (const scenario of ["failure", "subagent"]) { + const host = yield* controlled({ + [`${scenario}.md`]: { exit: 1, failedRecord: "Failed to authenticate: controlled refusal" }, + }); + const ran = yield* runControlled(host, scenario); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + // The host's own refusal reaches the operator; the report says only + // which required witness is missing. + expect(ran.stderr).toContain( + "Agent prompt failed: Failed to authenticate: controlled refusal", + ); + expect(JSON.stringify(report).includes("controlled refusal")).toBe(false); + expect([ran.status, report.result]).toEqual([1, "incomplete"]); + expect( + scenario === "failure" + ? report.scenarioChecks.attemptedWriteRefused + : report.scenarioChecks.nativeSubagent, + ).toEqual( + scenario === "failure" + ? { outcome: "failed", reason: "prompt-failed-before-write" } + : { outcome: "failed", reason: "experiment-not-executed" }, + ); + } + }); }); function usageWith(tokens: Record): PromptUsage { @@ -376,3 +604,173 @@ function usageWith(tokens: Record): PromptUsage { coverage: { finalReport: "received", outerPrompt: "partial", limitations: [] }, }; } + +/** What one controlled host does for each scenario document it is asked to run. */ +interface Behaviors { + readonly [document: string]: { + readonly exit?: number; + /** Write ACPX's live record of the Agent saying the ready marker. */ + readonly ready?: boolean; + /** Keep working until interrupted. */ + readonly hold?: boolean; + /** How long cleanup takes after an interrupt. */ + readonly cleanupMs?: number; + /** Make the run's report impossible to write. */ + readonly blockReport?: boolean; + /** Acknowledge one failed Prompt with this error before doing anything else. */ + readonly failedRecord?: string; + }; +} + +interface Controlled { + readonly root: string; + readonly host: string; + readonly log: string; + readonly home: string; +} + +/** + * A controlled stand-in for a compiled XMD host: it answers its version, logs + * each document it is asked to run, and behaves as the case says. It starts no + * agent and sends no Prompt. + */ +function* controlled(behaviors: Behaviors): Operation { + const root = yield* useRoot(); + const home = join(root, "home"); + const log = join(root, "invocations.jsonl"); + const host = join(root, "controlled-xmd.cjs"); + yield* ensureDir(home); + const config = JSON.stringify({ behaviors, log, home }); + yield* writeTextFile( + host, + `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const config = ${config}; +const args = process.argv.slice(2); +if (args.includes("--version")) { + console.log("controlled-host 0.0.0"); + process.exit(0); +} +const document = path.basename(args[1]); +const journal = args[args.indexOf("--journal") + 1]; +const prop = (name) => args[args.indexOf("--props-" + name) + 1]; +const note = (event) => fs.appendFileSync(config.log, JSON.stringify({ document, event }) + "\\n"); +note("started"); +const behavior = config.behaviors[document] ?? { exit: 0 }; +if (behavior.failedRecord) { + const event = { + type: "yield", + coroutineId: "root", + description: { type: "agent_prompt", name: "prompt:controlled#0", input: "controlled" }, + result: { + status: "ok", + value: { sequence: 0, agent: "codex", sessionKey: "controlled", status: "failed", text: "", error: { message: behavior.failedRecord } }, + }, + }; + fs.writeFileSync(journal, JSON.stringify(event) + "\\n"); +} +if (behavior.blockReport) { + fs.mkdirSync(path.join(path.dirname(journal), "report.json"), { recursive: true }); +} +if (behavior.ready) { + const sessions = path.join(config.home, ".acpx", "sessions"); + fs.mkdirSync(sessions, { recursive: true }); + const record = { + messages: [ + { User: { content: [{ Text: prop("first") }] } }, + { Agent: { content: [{ Text: "USAGE_CALIBRATION_READY" }] } }, + ], + }; + fs.writeFileSync(path.join(sessions, "controlled.json"), JSON.stringify(record)); +} +if (behavior.hold) { + process.on("SIGINT", () => { + note("interrupted"); + setTimeout(() => { + note("cleaned"); + process.exit(130); + }, behavior.cleanupMs ?? 0); + }); + setInterval(() => {}, 1000); +} else { + process.exit(behavior.exit ?? 0); +} +`, + ); + yield* until(chmod(host, 0o755)); + return { root, host, log, home }; +} + +function* documents(host: Controlled): Operation { + if (!(yield* exists(host.log))) { + return []; + } + return (yield* readTextFile(host.log)) + .split("\n") + .filter((line) => line.length > 0) + .map((line) => { + const { document, event } = JSON.parse(line); + return `${document}:${event}`; + }); +} + +/** Run the driver in this process against a controlled host, with short clocks. */ +function* runControlled( + host: Controlled, + scenario: string, + clocks: Partial = {}, +): Operation<{ + status: number; + stdout: string; + stderr: string; + output: string; + elapsedMs: number; +}> { + const output = join(host.root, `codex-compiled-${scenario}`); + const environment: CalibrationEnvironment = { + promptMs: 5_000, + admissionMs: 30_000, + slackMs: 0, + pollMs: 25, + home: host.home, + native: host.host, + ...clocks, + }; + const out: string[] = []; + const err: string[] = []; + const log = console.log; + const error = console.error; + console.log = (...values: unknown[]) => { + out.push(values.join(" ")); + }; + console.error = (...values: unknown[]) => { + err.push(values.join(" ")); + }; + const started = Date.now(); + try { + const status = yield* scoped(() => + calibrate( + [ + "--host=compiled", + `--xmd=${host.host}`, + "--agent=codex", + `--scenario=${scenario}`, + `--output=${output}`, + "--allow-paid", + ], + environment, + ), + ); + return { + status, + stdout: out.join("\n"), + stderr: err.join("\n"), + output, + elapsedMs: Date.now() - started, + }; + } finally { + console.log = log; + console.error = error; + } +} diff --git a/packages/cli/tests/fixtures/repl/issue-883/sidekick-harness.ts b/packages/cli/tests/fixtures/repl/issue-883/sidekick-harness.ts index 50d6b922c..180ea66f1 100644 --- a/packages/cli/tests/fixtures/repl/issue-883/sidekick-harness.ts +++ b/packages/cli/tests/fixtures/repl/issue-883/sidekick-harness.ts @@ -1,3 +1,4 @@ +import type { PromptUsageHost } from "@executablemd/acp"; import { useNodeReplProjectAccess } from "../../../../src/node-repl-project-access.ts"; import { testForSidekick } from "../../../../src/repl/sidekick/admission.ts"; import { pairedLocation } from "../../../../src/repl/sidekick/history.ts"; @@ -129,6 +130,8 @@ export function* useHarness( * only way to observe what Stop waits for. */ readonly fill?: (source: string) => Operation>; + /** The host's own build facts, for a case about the Prompt usage it retains. */ + readonly promptUsageHost?: PromptUsageHost; } = {}, ): Operation { // An entry's eval block compiles through a module this host has to supply, so @@ -176,6 +179,7 @@ export function* useHarness( createRuntime: fake.create, sessionStore: makeStore(), agentRegistry: makeRegistry({ [AGENT]: `${AGENT}-cmd` }), + ...(extra.promptUsageHost === undefined ? {} : { promptUsageHost: extra.promptUsageHost }), }, // The scripted agent, named to the profile rather than left to the // environment: the Sidekick assembly is built with it, and a conversation diff --git a/packages/cli/tests/prompt-usage-hosts.test.ts b/packages/cli/tests/prompt-usage-hosts.test.ts index bf1736355..56657d983 100644 --- a/packages/cli/tests/prompt-usage-hosts.test.ts +++ b/packages/cli/tests/prompt-usage-hosts.test.ts @@ -37,6 +37,9 @@ import { useWorkflowAgentProfile } from "../src/workflow-agent.ts"; import { createFakeAcp, makeStore } from "./support/fake-acp.ts"; import type { FakeAcp } from "./support/fake-acp.ts"; import { createRun, useStorageRoot, withStorage } from "./support/workflow-run.ts"; +import { records, useHarness } from "./fixtures/repl/issue-883/sidekick-harness.ts"; +import { parseDurableEvent } from "@executablemd/durable-streams"; +import { sleep } from "effection"; const HOST: PromptUsageHost = { host: "deno-source", xmd: { version: "0.0.0-hosts" } }; @@ -242,4 +245,53 @@ describe("Tier HU — Prompt usage through the host assemblies", () => { }); expect(usageIn(yield* stream.readAll())).toEqual([undefined]); }); + + it("HU7: a REPL entry retains usage through the entry publisher, and the Sidekick through its owner", function* () { + yield* useHostFiles(); + const held = yield* useHarness(undefined, { promptUsageHost: HOST }); + held.fake.script({ reply: "entry answered", usage: RETAINED }); + const entry = yield* held.session.submit( + '\n', + ); + expect(entry.ok ? "ok" : entry.error.message).toBe("ok"); + const entryUsage = yield* settledUsage(() => held.execution.stream.readAll()); + expect(entryUsage.length).toBe(1); + // The scripted agent has no traced adapter release, so its semantics stay + // unknown — the retained identities and provenance are the row's subject. + expect(entryUsage[0]?.providerMessageId).toBe("message-hosts"); + expect(entryUsage[0]?.provenance.host).toBe("deno-source"); + expect(entryUsage[0]?.coverage.limitations).toContain("source-semantics-unknown"); + + held.fake.script({ + reply: "sidekick answered", + usage: { ...RETAINED, promptMessageId: "message-sidekick" }, + }); + const turn = yield* held.owner.message("measure the conversation"); + expect(turn.ok ? "ok" : turn.error.message).toBe("ok"); + const chat = yield* settledUsage(function* () { + return (yield* records(held.execution)).flatMap((one) => { + if (one.kind !== "event") { + return []; + } + const parsed = parseDurableEvent(one.event); + return parsed.ok ? [parsed.value] : []; + }); + }); + expect(chat.map((usage) => usage?.providerMessageId)).toEqual(["message-sidekick"]); + expect(chat[0]?.provenance.host).toBe("deno-source"); + }); }); + +/** The usage each retained Prompt carries, once a history holds at least one. */ +function* settledUsage( + read: () => Operation, +): Operation<(PromptUsage | undefined)[]> { + for (let attempt = 0; attempt < 500; attempt++) { + const found = usageIn(yield* read()); + if (found.length > 0) { + return found; + } + yield* sleep(10); + } + throw new Error("no Prompt was retained"); +} diff --git a/packages/core/src/agent/usage.ts b/packages/core/src/agent/usage.ts index 805f91e64..56beb82c9 100644 --- a/packages/core/src/agent/usage.ts +++ b/packages/core/src/agent/usage.ts @@ -17,8 +17,7 @@ import type { Json } from "@executablemd/durable-streams"; import type { Operation } from "effection"; -import { inputValues } from "../secrets/inputs.ts"; -import { secretPolicy, scanSecrets } from "../secrets/policy.ts"; +import { retainsOptionalText } from "../secrets/policy.ts"; /** One build identity, where it was observed. */ export interface UsageBuild { @@ -448,34 +447,8 @@ export function canonicalUsage( return unavailableUsage(reported[0].provenance, USAGE_ASSOCIATION_AMBIGUOUS); } -/** - * Whether one optional string may be retained by the execution running now. - * - * Protected inputs are refused whatever the scanner setting, because that check - * is not the scanner's to waive. Doubt withholds: an absent or unauthentic - * policy, a finding and a scanner that failed all answer no. - */ -function* disclosable(value: string): Operation { - try { - const inputs = yield* inputValues(); - if (inputs !== undefined) { - if (inputs.contains(value)) { - return false; - } - const shown = inputs.display(value); - if (!shown.ok || shown.value !== value) { - return false; - } - } - const policy = yield* secretPolicy(); - if (!policy.enabled) { - return true; - } - return (yield* scanSecrets(value)).length === 0; - } catch { - return false; - } -} +/** A limitation is a fixed categorical name: lowercase words joined by hyphens. */ +const LIMITATION = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; /** * The part of a reported value this execution may retain. @@ -485,8 +458,10 @@ function* disclosable(value: string): Operation { * so one unsafe model name withholds that name rather than the whole account; * a reading whose only identity was a withheld model is dropped, because * keeping its numbers without the model would let it read as another view. - * Numbers, enumerations and the fixed limitation names carry nothing a host - * wrote. The journal's own gate still decides whether the record is persisted. + * Numbers and enumerations carry nothing a host wrote. Every supplied string — + * identities, builds, model names, currency and limitation names — is checked + * through the producing execution's own policy binding. The journal's own gate + * still decides whether the record is persisted. */ export function* retainablePromptUsage(usage: PromptUsage): Operation { let withheld = false; @@ -494,7 +469,7 @@ export function* retainablePromptUsage(usage: PromptUsage): Operation { } } +/** + * Whether one optional string the running execution observed may be retained. + * + * Answered by the execution's own authenticated policy binding and the + * protected inputs captured with it when the policy was selected — never by a + * same-name Context installed later, which could remove the protection by + * removing a value. A protected input is refused whether or not detection is + * enabled, because a scanner opt-out does not waive it. Doubt withholds: an + * absent or unauthentic binding, a projection that changes the value, a finding + * and a scanner that failed all answer `false`. + * + * Package-private, like {@link clearsLiveOutput}: it is a question about this + * execution's own optional observations, answered inside that execution. + */ +export function* retainsOptionalText(value: string): Operation { + let detection: ExecutionDetection; + try { + detection = yield* currentDetection(); + } catch { + return false; + } + const inputs = detection.inputs; + if (inputs !== undefined) { + if (inputs.contains(value)) { + return false; + } + const shown = inputs.display(value); + if (!shown.ok || shown.value !== value) { + return false; + } + } + if (!(detection instanceof EnabledDetection)) { + return true; + } + try { + return (yield* detection.scanner.scan(value)).length === 0; + } catch { + return false; + } +} + function projectInputEvent(received: DurableEvent, inputs: InputValues): DurableEvent { const serialized = parseDurableEvent(serializeDurableEvent(received)); if (!serialized.ok) { @@ -350,13 +397,13 @@ export function* useSecretDetection( return yield* disclose(parseJsonObject(safe.value)); }; if (requested === false) { - yield* CurrentDetection.set(new DisabledDetection()); + yield* CurrentDetection.set(new DisabledDetection(inputs)); return { journal: stream, disclose: projectedDisclosure(discloseFreely) }; } const create = (yield* CurrentScannerFactory.get()) ?? createSecretScanner; const scanner = create(); - yield* CurrentDetection.set(new EnabledDetection(scanner)); + yield* CurrentDetection.set(new EnabledDetection(scanner, inputs)); return { journal: guardWithSecretDetection(stream, scanner), disclose: projectedDisclosure(discloseThrough(scanner)), diff --git a/packages/core/tests/agent-usage.test.ts b/packages/core/tests/agent-usage.test.ts index b2d6dce98..77c5d697a 100644 --- a/packages/core/tests/agent-usage.test.ts +++ b/packages/core/tests/agent-usage.test.ts @@ -44,7 +44,10 @@ import { registerComponents } from "../src/components/registration.ts"; import type { SecretScanner } from "../src/secrets/scanner.ts"; import type { SecretFinding } from "../src/secrets/findings.ts"; import type { PromptUsage, PromptUsageReporter } from "../src/agent/usage.ts"; -import { useSecretScannerFactory } from "../src/secrets/policy.ts"; +import { useSecretDetection, useSecretScannerFactory } from "../src/secrets/policy.ts"; +import { attachInputOwner, createInputOwner } from "../src/secrets/inputs.ts"; +import { createContext } from "effection"; +import { retainablePromptUsage } from "../src/agent/usage.ts"; import { createSecretScanner } from "../src/secrets/scanner.ts"; import type { Json } from "../src/types.ts"; @@ -232,9 +235,17 @@ function stubFactory(trace: StubTrace, plans: () => TurnPlan): AgentProviderFact if (!authorized.ok) { throw authorized.error; } + // Established, as a provider that kept the conversation answers both + // when it is first opened and when its retained history is reopened. return yield* setup.finish( { sessionKey: `stub:${request.name ?? agent}`, cwd: "/stub" }, - { kind: "new" }, + { + kind: "existing", + // deno-lint-ignore require-yield + *configure(asked) { + return asked; + }, + }, ); }, // deno-lint-ignore require-yield @@ -597,6 +608,134 @@ function markerScanner(marker: string): () => SecretScanner { }); } +describe("Tier PU — producing-execution disclosure", () => { + it("PU-P5: a protected input stays withheld under scanner opt-out, whatever context is rebound", function* () { + yield* scoped(function* () { + const owner = yield* createInputOwner(); + const values = yield* attachInputOwner(owner); + const protectedText = "PU_PROTECTED_MARKER"; + values.protect({ coroutine: "root", index: 0, field: "code" }, protectedText); + // The producing execution's own policy, selected with its protected + // inputs and with the scanner explicitly turned off. + yield* useSecretDetection(false, new InMemoryStream(), values); + const reported: PromptUsage = { + ...versioned("protected"), + providerMessageId: protectedText, + coverage: { + finalReport: "received", + outerPrompt: "partial", + limitations: ["fixture-limit", protectedText, "Not A Category"], + }, + }; + const expected = { + message: undefined, + limitations: ["fixture-limit", "field-withheld"], + }; + const normal = yield* retainablePromptUsage(reported); + expect({ + message: normal.providerMessageId, + limitations: normal.coverage.limitations, + }).toEqual(expected); + // Removing the same-name contextual inputs removes nothing: the policy + // binding carries the inputs it was selected with. + yield* createContext("executablemd.secret-input.values").set(undefined); + const substituted = yield* retainablePromptUsage(reported); + expect({ + message: substituted.providerMessageId, + limitations: substituted.coverage.limitations, + }).toEqual(expected); + expect(substituted.requestId).toBe(reported.requestId); + }); + }); + + it("PU-P6: outside an execution's policy every supplied string is withheld", function* () { + const retained = yield* retainablePromptUsage(sampleUsage("orphan")); + expect(retained.requestId).toBe(undefined); + expect(retained.provenance).toEqual({ + host: "deno-source", + xmd: {}, + runtime: {}, + adapter: {}, + native: {}, + }); + expect(retained.readings.some((reading) => reading.source === "quota-model")).toBe(false); + expect(retained.coverage.limitations).toEqual(["field-withheld"]); + }); + + it("PU-P7: the final whole-event gate still refuses a record its fields passed one by one", function* () { + // Each optional string is clean alone; only the complete event holds both. + const scanner = (): SecretScanner => ({ + // deno-lint-ignore require-yield + *scan(content: string): Operation { + return content.includes("model-a") && content.includes("USD") + ? [ + { + ruleId: "test-rule", + messageId: "TEST_SECRET", + location: { line: 1, column: 1 }, + fingerprint: "test-fingerprint", + }, + ] + : []; + }, + }); + const stream = new InMemoryStream(); + const { result, events } = yield* runDocument( + "one\n", + stream, + stubFactory(newTrace(), always({ usage: () => sampleUsage("gated") })), + { scanner }, + ); + expect(result.ok).toBe(false); + expect(promptValues(events)).toEqual([]); + }); + + it("PU-P8: reopening a held conversation replays its usage without a provider or a new event", function* () { + const stream = new InMemoryStream(); + const installation = (trace: StubTrace, label: string): ExecutionInstallation => ({ + *install(): Operation { + yield* installAgentComponents({ + rootProvider: { + factory: stubFactory(trace, always({ usage: () => versioned(label) })), + options: { defaultAgent: AGENT, permissionMode: "deny-all" }, + }, + }); + }, + }); + function* converse(trace: StubTrace, label: string, text: string) { + return yield* scoped(function* () { + const opened = yield* useAgentConversation({ + history: stream, + id: "chat-replay", + agent: AGENT, + installations: [installation(trace, label)], + }); + if (!opened.ok) { + throw opened.error; + } + return yield* opened.value.prompt(text); + }); + } + const first = newTrace(); + const opened = yield* converse(first, "original", "hello"); + expect(opened.ok ? "ok" : opened.error.message).toBe("ok"); + const before = yield* stream.readAll(); + const [original] = promptValues(before); + + const again = newTrace(); + const reopened = yield* converse(again, "refreshed", "again"); + expect(reopened.ok ? "ok" : reopened.error.message).toBe("ok"); + const after = promptValues(yield* stream.readAll()); + // Only the new turn reached a provider or captured a receiver; the + // retained one answered from its record, unchanged and not appended again. + expect([again.subscriptions, again.acquisitions]).toEqual([1, 1]); + expect(after.length).toBe(2); + expect(after[0]).toEqual(original); + expect(parsePromptRecord(after[0])?.usage?.provenance.xmd.version).toBe("original"); + expect(parsePromptRecord(after[1])?.usage?.provenance.xmd.version).toBe("refreshed"); + }); +}); + describe("Tier PU — retained data", () => { it("PU-D1: a complete value round-trips, keeping zero, null and absent apart", function* () { const usage = sampleUsage("one"); diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index 143013eef..ff278fe47 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -608,6 +608,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ * exactly what happened. */ const COMPILED_BINARY: RuntimeExclusion[] = [ + { + path: "scripts/tests/prompt-usage-compiled.test.ts", + reason: + "runs the compiled `dist/xmd`, which only `deno compile` produces and no test shard builds, to read the Prompt usage the compiled entrypoint's own host assembly retains; it belongs beside the other compiled-binary evidence the `smoke` job builds for through README.md#Build", + issue: "https://github.com/taras/executable.md/issues/567", + }, { path: "scripts/tests/component-form-dispatch.test.ts", reason: diff --git a/scripts/tests/prompt-usage-compiled.test.ts b/scripts/tests/prompt-usage-compiled.test.ts new file mode 100644 index 000000000..c27b8968b --- /dev/null +++ b/scripts/tests/prompt-usage-compiled.test.ts @@ -0,0 +1,92 @@ +/** + * Prompt usage carried by the compiled binary (specs/acp-client-spec.md §Prompt usage). + * + * The compiled entrypoint states its own host facts, which is a different + * assembly from the source CLI's. This runs `dist/xmd` against the controlled + * native Claude fixture — no credentials, no model call — and reads the usage + * the binary retained in the journal it was asked to keep, and nothing where it + * was asked to keep none. + * + * It runs against `dist/xmd`, so `deno task build` has to have happened. + */ + +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { ensure } from "effection"; +import { ensureDir, exists, readdir, readTextFile, rm, writeTextFile } from "@effectionx/fs"; +import { exec } from "@effectionx/process"; +import { timebox } from "@effectionx/timebox"; +import type { ProcessResult } from "@effectionx/process"; +import type { Operation } from "effection"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { parsePromptRecord } from "@executablemd/core/host"; + +const ROOT = fileURLToPath(new URL("../../", import.meta.url)); +const BINARY = path.join(ROOT, "dist", "xmd"); +const FAKE_CLAUDE = path.join(ROOT, "packages/acp/tests/fixtures/fake-claude-cli.cjs"); + +function* runBinary(args: readonly string[], cwd: string): Operation { + const attempt = yield* timebox(120_000, function* () { + return yield* exec(BINARY, { + arguments: [...args], + cwd, + env: Object.fromEntries( + [...Object.entries(process.env), ["CLAUDE_CODE_EXECUTABLE", FAKE_CLAUDE]].flatMap( + ([name, value]) => (typeof value === "string" ? [[name, value]] : []), + ), + ), + }).join(); + }); + if (attempt.timeout) { + throw new Error(`the compiled binary timed out running ${args.join(" ")}`); + } + return attempt.value; +} + +describe("Prompt usage through the compiled binary", () => { + it("retains compiled-host usage only in the journal a run selected", function* () { + if (!(yield* exists(BINARY))) { + throw new Error("dist/xmd is missing; run `deno task build` first"); + } + const root = path.join(tmpdir(), `xmd-puc-${randomUUID()}`); + yield* ensureDir(root); + yield* ensure(() => rm(root, { recursive: true, force: true })); + yield* writeTextFile( + path.join(root, "doc.md"), + 'measure me\n', + ); + + const transient = yield* runBinary(["run", "doc.md"], root); + expect(transient.code).toBe(0); + expect((yield* readdir(root)).sort()).toEqual(["doc.md"]); + + const retained = yield* runBinary(["run", "doc.md", "--journal", "run.jsonl"], root); + expect(retained.code).toBe(0); + const records = (yield* readTextFile(path.join(root, "run.jsonl"))) + .split("\n") + .filter((line) => line.includes('"agent_prompt"')) + .map((line) => parsePromptRecord(JSON.parse(line).result.value)); + expect(records.length).toBe(1); + const usage = records[0]?.usage; + expect(usage?.provenance.host).toBe("compiled"); + expect(usage?.provenance.adapter.version).toBe("0.70.0"); + expect(typeof usage?.providerMessageId).toBe("string"); + expect(usage?.readings.find((reading) => reading.source === "response")).toEqual({ + source: "response", + scope: "main-loop", + kind: "tokens", + counter: "turn-total", + tokens: { + inputTokens: 1, + outputTokens: 1, + cachedReadTokens: 0, + cachedWriteTokens: 0, + totalTokens: 2, + }, + }); + }); +}); From 406cd392fe9f31d4fe19402704364e2c19123e70 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 07:22:39 -0500 Subject: [PATCH 06/10] test(core): drop a redundant scope from the protected-input usage witness Co-Authored-By: Claude Opus 5.5 --- packages/core/tests/agent-usage.test.ts | 70 ++++++++++++------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/packages/core/tests/agent-usage.test.ts b/packages/core/tests/agent-usage.test.ts index 77c5d697a..696e286cf 100644 --- a/packages/core/tests/agent-usage.test.ts +++ b/packages/core/tests/agent-usage.test.ts @@ -610,42 +610,40 @@ function markerScanner(marker: string): () => SecretScanner { describe("Tier PU — producing-execution disclosure", () => { it("PU-P5: a protected input stays withheld under scanner opt-out, whatever context is rebound", function* () { - yield* scoped(function* () { - const owner = yield* createInputOwner(); - const values = yield* attachInputOwner(owner); - const protectedText = "PU_PROTECTED_MARKER"; - values.protect({ coroutine: "root", index: 0, field: "code" }, protectedText); - // The producing execution's own policy, selected with its protected - // inputs and with the scanner explicitly turned off. - yield* useSecretDetection(false, new InMemoryStream(), values); - const reported: PromptUsage = { - ...versioned("protected"), - providerMessageId: protectedText, - coverage: { - finalReport: "received", - outerPrompt: "partial", - limitations: ["fixture-limit", protectedText, "Not A Category"], - }, - }; - const expected = { - message: undefined, - limitations: ["fixture-limit", "field-withheld"], - }; - const normal = yield* retainablePromptUsage(reported); - expect({ - message: normal.providerMessageId, - limitations: normal.coverage.limitations, - }).toEqual(expected); - // Removing the same-name contextual inputs removes nothing: the policy - // binding carries the inputs it was selected with. - yield* createContext("executablemd.secret-input.values").set(undefined); - const substituted = yield* retainablePromptUsage(reported); - expect({ - message: substituted.providerMessageId, - limitations: substituted.coverage.limitations, - }).toEqual(expected); - expect(substituted.requestId).toBe(reported.requestId); - }); + const owner = yield* createInputOwner(); + const values = yield* attachInputOwner(owner); + const protectedText = "PU_PROTECTED_MARKER"; + values.protect({ coroutine: "root", index: 0, field: "code" }, protectedText); + // The producing execution's own policy, selected with its protected + // inputs and with the scanner explicitly turned off. + yield* useSecretDetection(false, new InMemoryStream(), values); + const reported: PromptUsage = { + ...versioned("protected"), + providerMessageId: protectedText, + coverage: { + finalReport: "received", + outerPrompt: "partial", + limitations: ["fixture-limit", protectedText, "Not A Category"], + }, + }; + const expected = { + message: undefined, + limitations: ["fixture-limit", "field-withheld"], + }; + const normal = yield* retainablePromptUsage(reported); + expect({ + message: normal.providerMessageId, + limitations: normal.coverage.limitations, + }).toEqual(expected); + // Removing the same-name contextual inputs removes nothing: the policy + // binding carries the inputs it was selected with. + yield* createContext("executablemd.secret-input.values").set(undefined); + const substituted = yield* retainablePromptUsage(reported); + expect({ + message: substituted.providerMessageId, + limitations: substituted.coverage.limitations, + }).toEqual(expected); + expect(substituted.requestId).toBe(reported.requestId); }); it("PU-P6: outside an execution's policy every supplied string is withheld", function* () { From fdcc0ca447ed57bd1df62bdd153399f3760fd940 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 07:25:00 -0500 Subject: [PATCH 07/10] test(smoke): carry the compiled Prompt usage case as a smoke script The compiled-binary suite list is pinned by the runtime-exclusion test, so the compiled carriage case runs as a smoke script in the job that builds the binary, like the other binary checks. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- scripts/runtime-test-exclusions.ts | 6 -- scripts/smoke-prompt-usage.ts | 104 ++++++++++++++++++++ scripts/tests/prompt-usage-compiled.test.ts | 92 ----------------- 4 files changed, 105 insertions(+), 99 deletions(-) create mode 100644 scripts/smoke-prompt-usage.ts delete mode 100644 scripts/tests/prompt-usage-compiled.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 764bd5209..0494adb91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,7 +265,7 @@ jobs: # different assembly from the source CLI's, so its carriage is asserted # against the binary this job just built. - name: Prompt usage the compiled binary retains - run: deno test --allow-all --frozen scripts/tests/prompt-usage-compiled.test.ts + run: deno run --allow-all --frozen scripts/smoke-prompt-usage.ts # The entry host a REPL entry's file work is retained by is assembled by # `compiled.ts`, which is a different entrypoint from the one the source diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index ff278fe47..143013eef 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -608,12 +608,6 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ * exactly what happened. */ const COMPILED_BINARY: RuntimeExclusion[] = [ - { - path: "scripts/tests/prompt-usage-compiled.test.ts", - reason: - "runs the compiled `dist/xmd`, which only `deno compile` produces and no test shard builds, to read the Prompt usage the compiled entrypoint's own host assembly retains; it belongs beside the other compiled-binary evidence the `smoke` job builds for through README.md#Build", - issue: "https://github.com/taras/executable.md/issues/567", - }, { path: "scripts/tests/component-form-dispatch.test.ts", reason: diff --git a/scripts/smoke-prompt-usage.ts b/scripts/smoke-prompt-usage.ts new file mode 100644 index 000000000..5bc73d683 --- /dev/null +++ b/scripts/smoke-prompt-usage.ts @@ -0,0 +1,104 @@ +/** + * Prompt usage carried by the compiled binary (specs/acp-client-spec.md §Prompt usage). + * + * The compiled entrypoint states its own host facts, a different assembly from + * the source CLI's. This runs `dist/xmd` against the controlled native Claude + * fixture — no credentials, no model call — and reads the usage the binary + * retained in the journal it was asked to keep, and finds nothing where it was + * asked to keep none. + */ + +import { ensure, main } from "effection"; +import type { Operation } from "effection"; +import { exec } from "@effectionx/process"; +import type { ProcessResult } from "@effectionx/process"; +import { ensureDir, exists, readdir, readTextFile, rm, writeTextFile } from "@effectionx/fs"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import process from "node:process"; +import { parsePromptRecord } from "@executablemd/core/host"; + +const BINARY = path.join(process.cwd(), "dist", "xmd"); +const FAKE_CLAUDE = path.join(process.cwd(), "packages/acp/tests/fixtures/fake-claude-cli.cjs"); + +function fail(claim: string): never { + console.error(`prompt usage smoke: ${claim}`); + process.exit(1); +} + +function* runBinary(args: readonly string[], cwd: string): Operation { + return yield* exec(BINARY, { + arguments: [...args], + cwd, + env: Object.fromEntries( + [...Object.entries(process.env), ["CLAUDE_CODE_EXECUTABLE", FAKE_CLAUDE]].flatMap( + ([name, value]) => (typeof value === "string" ? [[name, value]] : []), + ), + ), + }).join(); +} + +await main(function* () { + if (!(yield* exists(BINARY))) { + fail("dist/xmd is missing; run `deno task build` first"); + } + const root = path.join(tmpdir(), `xmd-prompt-usage-${randomUUID()}`); + yield* ensureDir(root); + yield* ensure(() => rm(root, { recursive: true, force: true })); + yield* writeTextFile( + path.join(root, "doc.md"), + 'measure me\n', + ); + + const transient = yield* runBinary(["run", "doc.md"], root); + if (transient.code !== 0) { + fail(`a run without --journal exited ${transient.code}`); + } + const left = (yield* readdir(root)).sort(); + if (JSON.stringify(left) !== JSON.stringify(["doc.md"])) { + fail(`a run without --journal left ${JSON.stringify(left)}`); + } + + const retained = yield* runBinary(["run", "doc.md", "--journal", "run.jsonl"], root); + if (retained.code !== 0) { + fail(`a run with --journal exited ${retained.code}`); + } + const records = (yield* readTextFile(path.join(root, "run.jsonl"))) + .split("\n") + .filter((line) => line.includes('"agent_prompt"')) + .map((line) => parsePromptRecord(JSON.parse(line).result.value)); + if (records.length !== 1) { + fail(`the journal holds ${records.length} Prompt records, not 1`); + } + const usage = records[0]?.usage; + const response = usage?.readings.find((reading) => reading.source === "response"); + const expected = { + source: "response", + scope: "main-loop", + kind: "tokens", + counter: "turn-total", + tokens: { + inputTokens: 1, + outputTokens: 1, + cachedReadTokens: 0, + cachedWriteTokens: 0, + totalTokens: 2, + }, + }; + if (usage?.provenance.host !== "compiled") { + fail(`the retained host is ${usage?.provenance.host}, not compiled`); + } + if (usage.provenance.adapter.version !== "0.70.0") { + fail(`the retained adapter is ${usage.provenance.adapter.version}, not 0.70.0`); + } + if (typeof usage.providerMessageId !== "string") { + fail("the retained usage names no provider message"); + } + if (JSON.stringify(response) !== JSON.stringify(expected)) { + fail(`the retained response reading is ${JSON.stringify(response)}`); + } + console.log( + "prompt usage smoke: the compiled binary retained compiled-host usage only in its journal", + ); +}); diff --git a/scripts/tests/prompt-usage-compiled.test.ts b/scripts/tests/prompt-usage-compiled.test.ts deleted file mode 100644 index c27b8968b..000000000 --- a/scripts/tests/prompt-usage-compiled.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Prompt usage carried by the compiled binary (specs/acp-client-spec.md §Prompt usage). - * - * The compiled entrypoint states its own host facts, which is a different - * assembly from the source CLI's. This runs `dist/xmd` against the controlled - * native Claude fixture — no credentials, no model call — and reads the usage - * the binary retained in the journal it was asked to keep, and nothing where it - * was asked to keep none. - * - * It runs against `dist/xmd`, so `deno task build` has to have happened. - */ - -import { describe, it } from "@executablemd/test-support/bdd"; -import { expect } from "@executablemd/test-support/expect"; -import { ensure } from "effection"; -import { ensureDir, exists, readdir, readTextFile, rm, writeTextFile } from "@effectionx/fs"; -import { exec } from "@effectionx/process"; -import { timebox } from "@effectionx/timebox"; -import type { ProcessResult } from "@effectionx/process"; -import type { Operation } from "effection"; -import { randomUUID } from "node:crypto"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import process from "node:process"; -import { fileURLToPath } from "node:url"; -import { parsePromptRecord } from "@executablemd/core/host"; - -const ROOT = fileURLToPath(new URL("../../", import.meta.url)); -const BINARY = path.join(ROOT, "dist", "xmd"); -const FAKE_CLAUDE = path.join(ROOT, "packages/acp/tests/fixtures/fake-claude-cli.cjs"); - -function* runBinary(args: readonly string[], cwd: string): Operation { - const attempt = yield* timebox(120_000, function* () { - return yield* exec(BINARY, { - arguments: [...args], - cwd, - env: Object.fromEntries( - [...Object.entries(process.env), ["CLAUDE_CODE_EXECUTABLE", FAKE_CLAUDE]].flatMap( - ([name, value]) => (typeof value === "string" ? [[name, value]] : []), - ), - ), - }).join(); - }); - if (attempt.timeout) { - throw new Error(`the compiled binary timed out running ${args.join(" ")}`); - } - return attempt.value; -} - -describe("Prompt usage through the compiled binary", () => { - it("retains compiled-host usage only in the journal a run selected", function* () { - if (!(yield* exists(BINARY))) { - throw new Error("dist/xmd is missing; run `deno task build` first"); - } - const root = path.join(tmpdir(), `xmd-puc-${randomUUID()}`); - yield* ensureDir(root); - yield* ensure(() => rm(root, { recursive: true, force: true })); - yield* writeTextFile( - path.join(root, "doc.md"), - 'measure me\n', - ); - - const transient = yield* runBinary(["run", "doc.md"], root); - expect(transient.code).toBe(0); - expect((yield* readdir(root)).sort()).toEqual(["doc.md"]); - - const retained = yield* runBinary(["run", "doc.md", "--journal", "run.jsonl"], root); - expect(retained.code).toBe(0); - const records = (yield* readTextFile(path.join(root, "run.jsonl"))) - .split("\n") - .filter((line) => line.includes('"agent_prompt"')) - .map((line) => parsePromptRecord(JSON.parse(line).result.value)); - expect(records.length).toBe(1); - const usage = records[0]?.usage; - expect(usage?.provenance.host).toBe("compiled"); - expect(usage?.provenance.adapter.version).toBe("0.70.0"); - expect(typeof usage?.providerMessageId).toBe("string"); - expect(usage?.readings.find((reading) => reading.source === "response")).toEqual({ - source: "response", - scope: "main-loop", - kind: "tokens", - counter: "turn-total", - tokens: { - inputTokens: 1, - outputTokens: 1, - cachedReadTokens: 0, - cachedWriteTokens: 0, - totalTokens: 2, - }, - }); - }); -}); From ad77fded53a577749e03b245c0abcbdcfab9f49a Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 07:32:29 -0500 Subject: [PATCH 08/10] test(cli): keep the Prompt usage host rows independent of an installed Claude The rows serve every turn from the scripted runtime, but the embedded adapter's child environment resolved a real Claude executable from the machine first. The controlled leaf now states that child environment through the provider's existing agentProcessEnv dependency, so the rows run where no Claude is installed. The assemblies above the leaf are unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/cli/tests/prompt-usage-hosts.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/cli/tests/prompt-usage-hosts.test.ts b/packages/cli/tests/prompt-usage-hosts.test.ts index 56657d983..aab1e3014 100644 --- a/packages/cli/tests/prompt-usage-hosts.test.ts +++ b/packages/cli/tests/prompt-usage-hosts.test.ts @@ -19,6 +19,7 @@ import { ensureDir, rm } from "@effectionx/fs"; import { ensure } from "effection"; import { randomUUID } from "node:crypto"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { useHostFiles } from "@executablemd/runtime"; import { agentIdentityComponents, collect, retainedSource } from "@executablemd/core"; @@ -68,14 +69,32 @@ function stackAt(root: string, defaultAgent = "claude"): AgentStack { }; } +/** + * The controlled native leaf: the scripted runtime, and the child environment it + * would launch with. + * + * The host's own environment fact resolves a real Claude executable from this + * machine, which a turn served by the scripted runtime never starts. Stating the + * child environment here keeps the row independent of what the machine has + * installed, and everything above the leaf is the product's own assembly. + */ function leaf(stack: AgentStack, fake: FakeAcp): AcpxProviderDependencies { return { ...hostAcpDependencies(stack), createRuntime: fake.create, sessionStore: makeStore(), + // deno-lint-ignore require-yield + *agentProcessEnv() { + return { CLAUDE_CODE_EXECUTABLE: CONTROLLED_CLAUDE }; + }, }; } +/** The controlled native Claude fixture, named but never started by the scripted runtime. */ +const CONTROLLED_CLAUDE = fileURLToPath( + new URL("../../acp/tests/fixtures/fake-claude-cli.cjs", import.meta.url), +); + function usageIn(events: readonly DurableEvent[]): (PromptUsage | undefined)[] { return events.flatMap((event) => event.type === "yield" && From 855a0138da3a25654870ea66cbaab4df9d2142b2 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 10:18:44 -0500 Subject: [PATCH 09/10] fix(cli): require calibration cleanup and active cancellation evidence --- examples/agent-usage/README.md | 7 +- examples/agent-usage/calibrate.ts | 41 ++--- .../cli/tests/agent-usage-calibration.test.ts | 144 ++++++++++++++++-- 3 files changed, 163 insertions(+), 29 deletions(-) diff --git a/examples/agent-usage/README.md b/examples/agent-usage/README.md index c025049d1..2c5cda7f1 100644 --- a/examples/agent-usage/README.md +++ b/examples/agent-usage/README.md @@ -37,7 +37,12 @@ other scenario runs. A later invocation starts only after the one before it exit successfully: a host that ended unsuccessfully may have failed its own cleanup, so the resume continuation is never started after an unsuccessful seed. Each Prompt has 60 seconds of work, and no invocation is admitted after 300 seconds; either deadline interrupts the host's own work and -then waits for it to finish. +then waits for it to finish. A nonzero child exit leaves cleanup unproven, +including an expected Agent failure or cancellation, so calibration stays +incomplete. A readiness marker alone does not prove active work: cancellation +requires an unsettled journal before the interrupt and an acknowledged cancelled +Prompt afterwards. Missing that witness, or a Prompt that completed instead, +also leaves calibration incomplete. The scenario documents in `scenarios/` are what the host runs; read them to see exactly what is asked. The source host is `packages/cli/src/deno.ts` from this diff --git a/examples/agent-usage/calibrate.ts b/examples/agent-usage/calibrate.ts index 65b58ff5b..5b8389d16 100644 --- a/examples/agent-usage/calibrate.ts +++ b/examples/agent-usage/calibrate.ts @@ -49,7 +49,7 @@ interface Invocation { readonly journal: string; readonly prompts: number; readonly denyAll?: boolean; - /** Cancel once the live Agent output says this, while the work is still active. */ + /** Interrupt once the saved Agent output says this. */ readonly cancelOn?: string; } @@ -58,8 +58,9 @@ interface Joined { readonly started: boolean; readonly code?: number; readonly signal?: string; - /** The cancellation marker was seen while the work was still active, and cancellation was sent. */ + /** The cancellation marker was seen and an interrupt was sent. */ readonly cancelledAfterReady?: boolean; + readonly unacknowledgedAtInterrupt?: boolean; readonly deadline?: boolean; } @@ -168,7 +169,8 @@ function* measure( * The provider checkpoints a turn's session updates while it runs, so the * marker in the saved Agent text, in the record whose user message names this * run's private fixture, is the earliest evidence a host has that the Agent - * actually said it while its work is still active. Nothing is copied. + * actually said it. It does not establish whether the turn is still active. + * Nothing is copied. */ function* readyInAcpx(acpxRoot: string, fixture: string, since: number): Operation { const sessions = join(acpxRoot, "sessions"); @@ -278,6 +280,7 @@ function* invoke( yield* writeTextFile(stderrPath, stderr.slice(0, 8000)); }); let cancelledAfterReady = false; + let unacknowledgedAtInterrupt = false; let deadline = false; const workDeadline = Math.min( deadlineAt, @@ -290,6 +293,11 @@ function* invoke( yield* sleep(environment.pollMs); if (invocation.cancelOn !== undefined && !cancelledAfterReady && !deadline) { if (yield* readyInAcpx(join(environment.home, ".acpx"), invocation.cancelOn, started)) { + try { + unacknowledgedAtInterrupt = ((yield* readAcknowledged(invocation.journal)) ?? []).length === 0; + } catch { + // A partial journal line cannot prove the Prompt is unsettled. + } cancelledAfterReady = true; process.kill(child.pid, "SIGINT"); } @@ -309,7 +317,7 @@ function* invoke( // The join reports an absent code or signal as null. ...(typeof status.code === "number" ? { code: status.code } : {}), ...(typeof status.signal === "string" ? { signal: status.signal } : {}), - ...(invocation.cancelOn === undefined ? {} : { cancelledAfterReady }), + ...(invocation.cancelOn === undefined ? {} : { cancelledAfterReady, unacknowledgedAtInterrupt }), ...(deadline ? { deadline } : {}), }; }); @@ -525,16 +533,11 @@ function* runScenario( prompts, scenarioChecks: scenario, readback: acknowledged.length > 0 ? check("verified", "canonical-journal-read-back") : check("unavailable", "no-acknowledged-history"), - teardown: teardownCheck(selection, observed, plan.length), + teardown: teardownCheck(observed, plan.length), }; } -/** Whether this scenario's own expected path ends its one child unsuccessfully. */ -function expectsUnsuccessfulChild(selection: Selection): boolean { - return selection.scenario === "failure" || selection.scenario === "cancel"; -} - -function teardownCheck(selection: Selection, observed: Observed, planned: number): Check { +function teardownCheck(observed: Observed, planned: number): Check { if (observed.stopped === "unsuccessful-owner") { return check("failed", "unsuccessful-owner-stopped-admission"); } @@ -549,9 +552,7 @@ function teardownCheck(selection: Selection, observed: Observed, planned: number } // An expected Agent failure or cancellation exits nonzero like a cleanup // failure would: the owner joined, and its cleanup stays unproven. - return expectsUnsuccessfulChild(selection) - ? check("limited", "joined-cleanup-unproven") - : check("unavailable", "joined-cleanup-unproven"); + return check("unavailable", "joined-cleanup-unproven"); } function failedBeforeWork(entry: Acknowledged | undefined): boolean { @@ -652,9 +653,13 @@ function scenarioChecks( case "cancel": { const joined = joins[0]; return { - cancelledWhileActive: joined?.cancelledAfterReady === true - ? check("verified", "cancelled-after-ready-marker") - : check("failed", "ready-marker-not-observed-while-active"), + cancelledWhileActive: acknowledged.some((entry) => entry.record.status === "completed") + ? check("failed", "prompt-completed-before-cancellation") + : joined?.cancelledAfterReady !== true + ? check("failed", "ready-marker-not-observed-while-active") + : joined.unacknowledgedAtInterrupt === true && acknowledged.length === 1 && acknowledged[0].record.status === "cancelled" + ? check("verified", "acknowledged-cancellation-after-ready-marker") + : check("unavailable", "active-cancellation-unproven"), acknowledgement: acknowledged.length === 1 ? check("verified", `acknowledged-${acknowledged[0].record.status}-partial`) : check("verified", "no-acknowledged-record"), @@ -693,7 +698,7 @@ function scenarioChecks( function verdict(report: Report): "completed" | "limited" | "incomplete" { const required = Object.values(report.scenarioChecks); const teardown = report.teardown; - const joined = teardown.outcome === "verified" || (teardown.outcome === "limited" && teardown.reason === "joined-cleanup-unproven"); + const joined = teardown.outcome === "verified"; if (!joined || required.some((item) => item.outcome === "failed" || (item.outcome === "not_run" && item.reason !== "no-subagent-work"))) { return "incomplete"; } diff --git a/packages/cli/tests/agent-usage-calibration.test.ts b/packages/cli/tests/agent-usage-calibration.test.ts index 38bc1fb5a..8f9acee3e 100644 --- a/packages/cli/tests/agent-usage-calibration.test.ts +++ b/packages/cli/tests/agent-usage-calibration.test.ts @@ -518,7 +518,7 @@ describe("Tier UC — usage calibration driver", () => { expect(controlReport.reservedPrompts).toBe(2); }); - it("UC7: cancellation waits for the ready marker while work is active, then joins the owner", function* () { + it("UC7: an interrupt after the ready marker joins cleanup without inventing missing proof", function* () { const host = yield* controlled({ "cancel.md": { ready: true, hold: true, cleanupMs: 300 } }); const ran = yield* runControlled(host, "cancel"); expect(yield* documents(host)).toEqual([ @@ -528,18 +528,18 @@ describe("Tier UC — usage calibration driver", () => { ]); const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); expect(report.scenarioChecks.cancelledWhileActive).toEqual({ - outcome: "verified", - reason: "cancelled-after-ready-marker", + outcome: "unavailable", + reason: "active-cancellation-unproven", }); expect(report.scenarioChecks.acknowledgement.reason).toBe("no-acknowledged-record"); - expect(report.teardown).toEqual({ outcome: "limited", reason: "joined-cleanup-unproven" }); + expect(report.teardown).toEqual({ outcome: "unavailable", reason: "joined-cleanup-unproven" }); expect([ report.reservedPrompts, report.observedSubmissions, report.acknowledgedPrompts, ]).toEqual([1, 1, 0]); - // Submitted work with no acknowledged record is an accounting gap. - expect([ran.status, report.result]).toEqual([0, "limited"]); + // Neither a live process nor its expected exit proves active Prompt cancellation or cleanup success. + expect([ran.status, report.result]).toEqual([1, "incomplete"]); }); it("UC8: the admission deadline interrupts owned work, waits for its held cleanup and admits nothing later", function* () { @@ -595,6 +595,112 @@ describe("Tier UC — usage calibration driver", () => { }); }); +describe("Tier UC — cancellation calibration regressions", () => { + it("UC12: failed owning cleanup stays incomplete with or without an acknowledged cancellation", function* () { + for (const terminalOnInterrupt of [ + undefined, + "cancelled", + ] satisfies Behaviors[string]["terminalOnInterrupt"][]) { + const host = yield* controlled({ + "cancel.md": { + ready: true, + hold: true, + terminalOnInterrupt, + exit: 1, + cleanupFailure: true, + }, + }); + const ran = yield* runControlled(host, "cancel"); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(yield* documents(host)).toEqual([ + "cancel.md:started", + "cancel.md:interrupted", + "cancel.md:cleanup-failed", + ]); + expect([ran.status, report.result]).toEqual([1, "incomplete"]); + expect(report.teardown).toEqual({ + outcome: "unavailable", + reason: "joined-cleanup-unproven", + }); + expect(report.scenarioChecks.acknowledgement.reason).toBe( + terminalOnInterrupt === undefined + ? "no-acknowledged-record" + : "acknowledged-cancelled-partial", + ); + expect(yield* readTextFile(join(ran.output, "support/child-0-stderr.txt"))).toContain( + "controlled owning-cleanup failure", + ); + expect(ran.stdout).not.toContain("Calibration completed"); + } + }); + + it("UC13: a completed Prompt waiting on process cleanup is not cancelled while active", function* () { + const host = yield* controlled({ + "cancel.md": { ready: true, hold: true, completedBeforeReady: true, exit: 0 }, + }); + const ran = yield* runControlled(host, "cancel"); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(yield* documents(host)).toEqual([ + "cancel.md:started", + "cancel.md:interrupted", + "cancel.md:cleaned", + ]); + expect([ran.status, report.result]).toEqual([1, "incomplete"]); + expect(report.teardown.outcome).toBe("verified"); + expect(report.scenarioChecks.cancelledWhileActive).toEqual({ + outcome: "failed", + reason: "prompt-completed-before-cancellation", + }); + expect(report.scenarioChecks.acknowledgement.reason).toBe("acknowledged-completed-partial"); + }); + + it("UC14: acknowledged active cancellation with successful owner cleanup remains a valid experiment", function* () { + const host = yield* controlled({ + "cancel.md": { + ready: true, + hold: true, + terminalOnInterrupt: "cancelled", + cleanupMs: 200, + exit: 0, + }, + }); + const ran = yield* runControlled(host, "cancel"); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(yield* documents(host)).toEqual([ + "cancel.md:started", + "cancel.md:interrupted", + "cancel.md:cleaned", + ]); + expect(ran.elapsedMs).toBeGreaterThanOrEqual(200); + expect([ran.status, report.result]).toEqual([0, "limited"]); + expect(report.teardown).toEqual({ outcome: "verified", reason: "owner-cleanup-succeeded" }); + expect(report.scenarioChecks.cancelledWhileActive).toEqual({ + outcome: "verified", + reason: "acknowledged-cancellation-after-ready-marker", + }); + expect(report.scenarioChecks.acknowledgement.reason).toBe("acknowledged-cancelled-partial"); + expect(report.prompts[0].status).toBe("cancelled"); + expect(report.prompts[0].checks.usage.outcome).toBe("unavailable"); + }); + + it("UC15: expected Agent failure preserves its outcome but cannot prove failed owner cleanup", function* () { + for (const exit of [1, 0]) { + const host = yield* controlled({ + "failure.md": { failedRecord: "controlled Agent failure", deniedWrite: true, exit }, + }); + const ran = yield* runControlled(host, "failure"); + const report = JSON.parse(yield* readTextFile(join(ran.output, "report.json"))); + expect(yield* documents(host)).toEqual(["failure.md:started"]); + expect(report.prompts[0].status).toBe("failed"); + expect(report.scenarioChecks.attemptedWriteRefused.outcome).toBe("verified"); + expect(report.scenarioChecks.nothingWritten.outcome).toBe("verified"); + expect([ran.status, report.result, report.teardown.outcome]).toEqual( + exit === 0 ? [0, "limited", "verified"] : [1, "incomplete", "unavailable"], + ); + } + }); +}); + function usageWith(tokens: Record): PromptUsage { return { provenance: { host: "compiled", xmd: {}, runtime: {}, adapter: {} }, @@ -619,6 +725,10 @@ interface Behaviors { readonly blockReport?: boolean; /** Acknowledge one failed Prompt with this error before doing anything else. */ readonly failedRecord?: string; + readonly completedBeforeReady?: boolean; + readonly terminalOnInterrupt?: "cancelled"; + readonly cleanupFailure?: boolean; + readonly deniedWrite?: boolean; }; } @@ -658,18 +768,26 @@ const prop = (name) => args[args.indexOf("--props-" + name) + 1]; const note = (event) => fs.appendFileSync(config.log, JSON.stringify({ document, event }) + "\\n"); note("started"); const behavior = config.behaviors[document] ?? { exit: 0 }; -if (behavior.failedRecord) { +function acknowledge(status, error) { const event = { type: "yield", coroutineId: "root", description: { type: "agent_prompt", name: "prompt:controlled#0", input: "controlled" }, result: { status: "ok", - value: { sequence: 0, agent: "codex", sessionKey: "controlled", status: "failed", text: "", error: { message: behavior.failedRecord } }, + value: { sequence: 0, agent: "codex", sessionKey: "controlled", status, text: "", ...(behavior.deniedWrite ? { permissions: [{ + toolCallId: "controlled-write", kind: "edit", options: [], outcome: { outcome: "cancelled" }, + }] } : {}), ...(error ? { error: { message: error } } : {}) }, }, }; fs.writeFileSync(journal, JSON.stringify(event) + "\\n"); } +if (behavior.failedRecord) { + acknowledge("failed", behavior.failedRecord); +} +if (behavior.completedBeforeReady) { + acknowledge("completed"); +} if (behavior.blockReport) { fs.mkdirSync(path.join(path.dirname(journal), "report.json"), { recursive: true }); } @@ -687,9 +805,15 @@ if (behavior.ready) { if (behavior.hold) { process.on("SIGINT", () => { note("interrupted"); + if (behavior.terminalOnInterrupt) { + acknowledge(behavior.terminalOnInterrupt); + } setTimeout(() => { - note("cleaned"); - process.exit(130); + note(behavior.cleanupFailure ? "cleanup-failed" : "cleaned"); + if (behavior.cleanupFailure) { + console.error("controlled owning-cleanup failure"); + } + process.exit(behavior.exit ?? 130); }, behavior.cleanupMs ?? 0); }); setInterval(() => {}, 1000); From 4a31a8e132ca97d4a013bb886d51951a370918c8 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Sat, 7 Nov 2026 10:24:59 -0500 Subject: [PATCH 10/10] test: record runtime weights for Prompt usage corpus --- test-weights.json | 2875 +++++++++++++++++++++++---------------------- 1 file changed, 1452 insertions(+), 1423 deletions(-) diff --git a/test-weights.json b/test-weights.json index 445070a4c..3f182fad7 100644 --- a/test-weights.json +++ b/test-weights.json @@ -1,8 +1,8 @@ { "version": 1, "source": { - "commit": "0c6b488f4af7dea23953a38181b127d72d9e7cac", - "runUrl": "https://github.com/taras/executable.md/actions/runs/42222287812", + "commit": "57582cea5468376c7b12f4c51375811c5fc32cca", + "runUrl": "https://github.com/taras/executable.md/actions/runs/42374830042", "attempt": 1, "runner": "ubuntu-latest", "deno": "deno 2.9.5 (stable, release, x86_64-unknown-linux-gnu)", @@ -11,1441 +11,1470 @@ }, "runtimes": { "deno": { - "packages/acp/tests/acceptance.test.ts": 3807, - "packages/acp/tests/acpx-checkpoint-meta.test.ts": 9441, - "packages/acp/tests/acpx-materialization.test.ts": 26592, - "packages/acp/tests/acpx-session-save.test.ts": 3030, - "packages/acp/tests/acpx-transient-env.test.ts": 3119, - "packages/acp/tests/adapter-materialization.test.ts": 64052, - "packages/acp/tests/adapter-protocol.test.ts": 34827, - "packages/acp/tests/agent-certification-report.test.ts": 2794, - "packages/acp/tests/agent-certification.test.ts": 36314, - "packages/acp/tests/codex-surfaces.test.ts": 1157, - "packages/acp/tests/host-conversation.test.ts": 10586, - "packages/acp/tests/loaded-copy.test.ts": 6345, - "packages/acp/tests/native-launch.test.ts": 3858, - "packages/acp/tests/permission-bridge.test.ts": 3121, - "packages/acp/tests/provider.test.ts": 4099, - "packages/acp/tests/released-runtime.test.ts": 44975, - "packages/acp/tests/serial-queue.test.ts": 1027, - "packages/acp/tests/session-key.test.ts": 3010, - "packages/acp/tests/session-route.test.ts": 3211, - "packages/acp/tests/terminal-screen.test.ts": 1094, - "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 13694, - "packages/cli/tests/adversarial-artifact-return.test.ts": 28715, - "packages/cli/tests/adversarial-delivery.test.ts": 28641, - "packages/cli/tests/adversarial-friction.test.ts": 86398, - "packages/cli/tests/adversarial-implementation.test.ts": 93423, - "packages/cli/tests/adversarial-launch-policy.test.ts": 64154, - "packages/cli/tests/adversarial-preparation.test.ts": 147193, - "packages/cli/tests/adversarial-production-role.test.ts": 53123, - "packages/cli/tests/adversarial-session-continuity.test.ts": 17212, - "packages/cli/tests/adversarial-workflow.test.ts": 65818, - "packages/cli/tests/agent-adapters.test.ts": 5053, - "packages/cli/tests/agent-certification.test.ts": 206327, - "packages/cli/tests/agent-cli.test.ts": 33025, - "packages/cli/tests/agent-config.test.ts": 2942, - "packages/cli/tests/agent-options-cli.test.ts": 19637, - "packages/cli/tests/agent-options.test.ts": 3003, - "packages/cli/tests/agent-session-coordinator.test.ts": 3549, - "packages/cli/tests/cli-help.test.ts": 40467, - "packages/cli/tests/command.test.ts": 11649, - "packages/cli/tests/compiled-upgrade.test.ts": 8942, - "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 7188, - "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 130720, - "packages/cli/tests/document-suites/props/props-markdown.test.ts": 6761, - "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 9622, - "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 8156, - "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 12249, - "packages/cli/tests/evaluate-component.test.ts": 8821, - "packages/cli/tests/evaluate-workflow.test.ts": 4130, - "packages/cli/tests/fetch-cli.test.ts": 2702, - "packages/cli/tests/file-stream.test.ts": 1241, - "packages/cli/tests/github-zero-effect.test.ts": 5091, - "packages/cli/tests/inline-cli.test.ts": 24727, - "packages/cli/tests/launch-timeout.test.ts": 11090, - "packages/cli/tests/packaged-document.test.ts": 3725, - "packages/cli/tests/plan-args.test.ts": 1086, - "packages/cli/tests/plan-cli.test.ts": 80736, - "packages/cli/tests/plan-command-document.test.ts": 7714, - "packages/cli/tests/plan-component.test.ts": 9330, - "packages/cli/tests/plan-host-acts.test.ts": 4103, - "packages/cli/tests/plan.test.ts": 11321, - "packages/cli/tests/plugin-cli.test.ts": 30386, - "packages/cli/tests/plugin-host.test.ts": 4681, - "packages/cli/tests/plugin-modules.test.ts": 5936, - "packages/cli/tests/plugin-selection.test.ts": 1020, - "packages/cli/tests/process-retention.test.ts": 4308, - "packages/cli/tests/props-cli.test.ts": 22707, - "packages/cli/tests/props-schema.test.ts": 3167, - "packages/cli/tests/props-sources.test.ts": 1589, - "packages/cli/tests/protected-input-cli.test.ts": 8483, - "packages/cli/tests/protected-input-forms.test.ts": 5774, - "packages/cli/tests/pull-request-read-fork.test.ts": 5590, - "packages/cli/tests/repl-admission.test.ts": 1013, - "packages/cli/tests/repl-agent-execution.test.ts": 6001, - "packages/cli/tests/repl-agent-interface.test.ts": 31533, - "packages/cli/tests/repl-agent-journey.test.ts": 16612, - "packages/cli/tests/repl-asset-import.test.ts": 8966, - "packages/cli/tests/repl-boundaries.test.ts": 6082, - "packages/cli/tests/repl-caret.test.ts": 5197, - "packages/cli/tests/repl-composition.test.ts": 6889, - "packages/cli/tests/repl-entries.test.ts": 8809, - "packages/cli/tests/repl-exec-output.test.ts": 7988, - "packages/cli/tests/repl-execution.test.ts": 4746, - "packages/cli/tests/repl-exit-pty.test.ts": 5235, - "packages/cli/tests/repl-field-ownership.test.ts": 5143, - "packages/cli/tests/repl-focus-navigation.test.ts": 1201, - "packages/cli/tests/repl-focus-regions.test.ts": 11952, - "packages/cli/tests/repl-forms.test.ts": 7665, - "packages/cli/tests/repl-git-observations.test.ts": 6228, - "packages/cli/tests/repl-headless-attach.test.ts": 43318, - "packages/cli/tests/repl-headless-clients.test.ts": 50178, - "packages/cli/tests/repl-headless-submit-process.test.ts": 13628, - "packages/cli/tests/repl-headless-submit.test.ts": 21252, - "packages/cli/tests/repl-headless.test.ts": 13122, - "packages/cli/tests/repl-history.test.ts": 4838, - "packages/cli/tests/repl-include-pty.test.ts": 23497, - "packages/cli/tests/repl-issues-pty.test.ts": 9699, - "packages/cli/tests/repl-issues.test.ts": 9434, - "packages/cli/tests/repl-journey.test.ts": 30578, - "packages/cli/tests/repl-layout-admission.test.ts": 9994, - "packages/cli/tests/repl-lifecycle.test.ts": 4045, - "packages/cli/tests/repl-model.test.ts": 4284, - "packages/cli/tests/repl-mounts-pty.test.ts": 23063, - "packages/cli/tests/repl-page.test.ts": 7142, - "packages/cli/tests/repl-plugin.test.ts": 6468, - "packages/cli/tests/repl-presentation-style.test.ts": 26860, - "packages/cli/tests/repl-project-files.test.ts": 12899, - "packages/cli/tests/repl-project-recovery.test.ts": 9410, - "packages/cli/tests/repl-push-pty.test.ts": 19008, - "packages/cli/tests/repl-push.test.ts": 8977, - "packages/cli/tests/repl-repositories-pty.test.ts": 37482, - "packages/cli/tests/repl-repositories.test.ts": 9061, - "packages/cli/tests/repl-retained-history.test.ts": 5513, - "packages/cli/tests/repl-route.test.ts": 5752, - "packages/cli/tests/repl-sidekick-acp.test.ts": 203277, - "packages/cli/tests/repl-sidekick-configuration.test.ts": 18415, - "packages/cli/tests/repl-sidekick-controls.test.ts": 36787, - "packages/cli/tests/repl-sidekick-dispatch.test.ts": 10716, - "packages/cli/tests/repl-sidekick-journey.test.ts": 24214, - "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 15123, - "packages/cli/tests/repl-sidekick-output.test.ts": 1005, - "packages/cli/tests/repl-sidekick-presentation.test.ts": 50720, - "packages/cli/tests/repl-sidekick-recovery.test.ts": 124464, - "packages/cli/tests/repl-sidekick-storage.test.ts": 3483, - "packages/cli/tests/repl-sidekick-testing.test.ts": 17838, - "packages/cli/tests/repl-sidekick-typing.test.ts": 15732, - "packages/cli/tests/repl-source-reading.test.ts": 6524, - "packages/cli/tests/repl-terminal.test.ts": 12941, - "packages/cli/tests/repl-unfinished-entry.test.ts": 12516, - "packages/cli/tests/repl-wheel-frame.test.ts": 5555, - "packages/cli/tests/repl-wheel-input.test.ts": 5074, - "packages/cli/tests/repl-wheel-migration.test.ts": 35127, - "packages/cli/tests/repl-wheel-scroll.test.ts": 29162, - "packages/cli/tests/retained-mount-files.test.ts": 4186, - "packages/cli/tests/retained-mount-records.test.ts": 3883, - "packages/cli/tests/retained-repository-git.test.ts": 12802, - "packages/cli/tests/retained-repository-history.test.ts": 5291, - "packages/cli/tests/retained-repository-lifecycle.test.ts": 8635, - "packages/cli/tests/retained-repository-local.test.ts": 13559, - "packages/cli/tests/retained-repository-observations-deno.test.ts": 7354, - "packages/cli/tests/retained-repository-observations.test.ts": 7953, - "packages/cli/tests/retained-repository-pull-request-records.test.ts": 3916, - "packages/cli/tests/retained-repository-pull-request.test.ts": 21703, - "packages/cli/tests/retained-repository-push-records.test.ts": 4022, - "packages/cli/tests/retained-repository-push.test.ts": 21350, - "packages/cli/tests/retained-repository-records.test.ts": 3363, - "packages/cli/tests/retained-repository-selection.test.ts": 21590, - "packages/cli/tests/run-composition-deno.test.ts": 6296, - "packages/cli/tests/run-composition-nested.test.ts": 18057, - "packages/cli/tests/run-composition.test.ts": 6546, - "packages/cli/tests/run-deadline.test.ts": 7492, - "packages/cli/tests/run-profile.test.ts": 3462, - "packages/cli/tests/run-timeouts.test.ts": 10606, - "packages/cli/tests/secret-detection-cli.test.ts": 14180, - "packages/cli/tests/service-document.test.ts": 4056, - "packages/cli/tests/service-host.test.ts": 4030, - "packages/cli/tests/session-launch-cli.test.ts": 8702, - "packages/cli/tests/session-ownership-cli.test.ts": 9286, - "packages/cli/tests/stdin-cli.test.ts": 30123, - "packages/cli/tests/stdout-delivery.test.ts": 1072, - "packages/cli/tests/syntax-cli.test.ts": 27283, - "packages/cli/tests/targets-cli.test.ts": 23280, - "packages/cli/tests/test-root-profile.test.ts": 6118, - "packages/cli/tests/test-target.test.ts": 28043, - "packages/cli/tests/testing-activation.test.ts": 3521, - "packages/cli/tests/testing-execution-host-native.test.ts": 6800, - "packages/cli/tests/testing-execution-host.test.ts": 53300, - "packages/cli/tests/upgrade-cli.test.ts": 16122, - "packages/cli/tests/upgrade-command-document.test.ts": 5726, - "packages/cli/tests/upgrade-output.test.ts": 3572, - "packages/cli/tests/value-root.test.ts": 13079, - "packages/cli/tests/verbose-component.test.ts": 3663, - "packages/cli/tests/workflow-activation.test.ts": 48205, - "packages/cli/tests/workflow-agent-sessions.test.ts": 29567, - "packages/cli/tests/workflow-agent.test.ts": 11313, - "packages/cli/tests/workflow-cli.test.ts": 86364, - "packages/cli/tests/workflow-crash.test.ts": 75186, - "packages/cli/tests/workflow-declaration.test.ts": 3279, - "packages/cli/tests/workflow-export-publication.test.ts": 14237, - "packages/cli/tests/workflow-fetch.test.ts": 5967, - "packages/cli/tests/workflow-fork.test.ts": 64763, - "packages/cli/tests/workflow-host.test.ts": 11283, - "packages/cli/tests/workflow-inspection.test.ts": 90332, - "packages/cli/tests/workflow-installation.test.ts": 4497, - "packages/cli/tests/workflow-lifecycle-control.test.ts": 16112, - "packages/cli/tests/workflow-retention.test.ts": 18530, - "packages/cli/tests/workflow-suspension.test.ts": 8940, - "packages/cli/tests/workspace-mounts.test.ts": 8714, - "packages/code-review-agent/tests/doctor.test.ts": 1000, - "packages/code-review-agent/tests/parse-diagnostics.test.ts": 1028, - "packages/code-review-agent/tests/parse-diff.test.ts": 1000, - "packages/code-review-agent/tests/parse-doctor.test.ts": 3008, - "packages/code-review-agent/tests/parse-oxlint.test.ts": 986, - "packages/code-review-agent/tests/policy.test.ts": 1004, - "packages/core/tests/agent-api.test.ts": 2635, - "packages/core/tests/agent-components.test.ts": 3319, - "packages/core/tests/agent-conversation.test.ts": 4517, - "packages/core/tests/agent-function-components.test.ts": 4743, - "packages/core/tests/agent-session-launch.test.ts": 4628, - "packages/core/tests/agent-session-setup.test.ts": 2703, - "packages/core/tests/agent-session-use-loaded-copy.test.ts": 2714, - "packages/core/tests/agent-session-use.test.ts": 2731, - "packages/core/tests/agent-stack.test.ts": 3231, - "packages/core/tests/all.test.ts": 4490, - "packages/core/tests/answer-identity.test.ts": 2860, - "packages/core/tests/answers-component.test.ts": 3632, - "packages/core/tests/answers-expansion-recursion.test.ts": 2861, - "packages/core/tests/capture-errors.test.ts": 2960, - "packages/core/tests/capture-props.test.ts": 2786, - "packages/core/tests/cli-journal.test.ts": 8606, - "packages/core/tests/code-block-component.test.ts": 3900, - "packages/core/tests/compiler-boundary.test.ts": 2933, - "packages/core/tests/component-api.test.ts": 2703, - "packages/core/tests/component-expansion.test.ts": 3083, - "packages/core/tests/component-invocation-hooks.test.ts": 3404, - "packages/core/tests/component-module-loader.test.ts": 4421, - "packages/core/tests/component-registration.test.ts": 3748, - "packages/core/tests/component-returns.test.ts": 4242, - "packages/core/tests/component-schema-conformance.test.ts": 2875, - "packages/core/tests/config-api.test.ts": 2915, - "packages/core/tests/construct-error-observation.test.ts": 2952, - "packages/core/tests/contextual-cwd.test.ts": 4554, - "packages/core/tests/daemon-integration.test.ts": 3592, - "packages/core/tests/daemon.test.ts": 2577, - "packages/core/tests/declared-markdown-component.test.ts": 3971, - "packages/core/tests/document-output-api.test.ts": 991, - "packages/core/tests/document-target-execution.test.ts": 4913, - "packages/core/tests/document-targets.test.ts": 3064, - "packages/core/tests/document-validation.test.ts": 3961, - "packages/core/tests/documentation-index.test.ts": 2827, - "packages/core/tests/each.test.ts": 3034, - "packages/core/tests/elicit-component.test.ts": 3361, - "packages/core/tests/ephemeral-service.test.ts": 3274, - "packages/core/tests/eval-bindings.test.ts": 3028, - "packages/core/tests/eval-context.test.ts": 1562, - "packages/core/tests/eval-durable.test.ts": 3068, - "packages/core/tests/eval-error-mode.test.ts": 3222, - "packages/core/tests/eval-interpolate.test.ts": 2816, - "packages/core/tests/eval-middleware.test.ts": 2924, - "packages/core/tests/eval-persist.test.ts": 3189, - "packages/core/tests/eval-return.test.ts": 3031, - "packages/core/tests/eval-scope.test.ts": 2862, - "packages/core/tests/eval-timeout.test.ts": 2943, - "packages/core/tests/eval-transform.test.ts": 2698, - "packages/core/tests/evaluate-component.test.ts": 4377, - "packages/core/tests/evaluate-loaded-copy.test.ts": 3004, - "packages/core/tests/evaluate-provider-lifetime.test.ts": 3051, - "packages/core/tests/evaluation-profile.test.ts": 2836, - "packages/core/tests/exec-timeout.test.ts": 2931, - "packages/core/tests/execute.test.ts": 3587, - "packages/core/tests/execution-init.test.ts": 3148, - "packages/core/tests/execution-protocol.test.ts": 3895, - "packages/core/tests/expand.test.ts": 3897, - "packages/core/tests/expansion-identity.test.ts": 3650, - "packages/core/tests/expansion-metadata.test.ts": 3279, - "packages/core/tests/expression-props.test.ts": 3824, - "packages/core/tests/fail-component.test.ts": 3630, - "packages/core/tests/failure-printing.test.ts": 3601, - "packages/core/tests/fatal-cause.test.ts": 3173, - "packages/core/tests/fetch-component.test.ts": 4421, - "packages/core/tests/file-component.test.ts": 4537, - "packages/core/tests/file-delete-component.test.ts": 3057, - "packages/core/tests/files-fatal.test.ts": 3651, - "packages/core/tests/filesystem-placement.test.ts": 3636, - "packages/core/tests/fixed-filesystem-components.test.ts": 3174, - "packages/core/tests/foreground-exec.test.ts": 3662, - "packages/core/tests/frontmatter.test.ts": 2861, - "packages/core/tests/function-components.test.ts": 3708, - "packages/core/tests/generated-composition.test.ts": 3412, - "packages/core/tests/generated-owner-metadata.test.ts": 3059, - "packages/core/tests/generated-selection.test.ts": 3565, - "packages/core/tests/generated-xmd-preview.test.ts": 2937, - "packages/core/tests/generated-xmd-root.test.ts": 3743, - "packages/core/tests/generated-xmd.test.ts": 4153, - "packages/core/tests/glob-api.test.ts": 1226, - "packages/core/tests/glob-component.test.ts": 3536, - "packages/core/tests/guarded-journal.test.ts": 3033, - "packages/core/tests/has-content.test.ts": 2736, - "packages/core/tests/heal.test.ts": 3122, - "packages/core/tests/host-generated-root.test.ts": 4083, - "packages/core/tests/if.test.ts": 3731, - "packages/core/tests/inline-root.test.ts": 3451, - "packages/core/tests/invocation-failures.test.ts": 2903, - "packages/core/tests/invocation-identity.test.ts": 3827, - "packages/core/tests/invocation-scope.test.ts": 2992, - "packages/core/tests/journal-source-position.test.ts": 3076, - "packages/core/tests/json-component.test.ts": 3122, - "packages/core/tests/json.test.ts": 2663, - "packages/core/tests/let.test.ts": 3202, - "packages/core/tests/loaded-copy-files.test.ts": 3261, - "packages/core/tests/loop.test.ts": 4085, - "packages/core/tests/named-slots.test.ts": 4033, - "packages/core/tests/output-error-mode.test.ts": 4285, - "packages/core/tests/output-normalize.test.ts": 1145, - "packages/core/tests/output-terminal.test.ts": 1193, - "packages/core/tests/parse-components.test.ts": 3318, - "packages/core/tests/plan-response.test.ts": 1115, - "packages/core/tests/plugin-api.test.ts": 3214, - "packages/core/tests/plugin.test.ts": 3196, - "packages/core/tests/props-binding.test.ts": 3443, - "packages/core/tests/protected-content.test.ts": 3037, - "packages/core/tests/protected-input-replay.test.ts": 3843, - "packages/core/tests/protected-input.test.ts": 4222, - "packages/core/tests/query-component.test.ts": 3515, - "packages/core/tests/registered-printed-errors.test.ts": 3133, - "packages/core/tests/replay-stream.test.ts": 1105, - "packages/core/tests/retain.test.ts": 3342, - "packages/core/tests/root-composition.test.ts": 3428, - "packages/core/tests/root-props.test.ts": 3462, - "packages/core/tests/root-provider.test.ts": 3483, - "packages/core/tests/sample-component.test.ts": 3797, - "packages/core/tests/scanner.test.ts": 2940, - "packages/core/tests/scope-local.test.ts": 973, - "packages/core/tests/secret-detection.test.ts": 3938, - "packages/core/tests/secret-files.test.ts": 1495, - "packages/core/tests/secret-rules.test.ts": 1447, - "packages/core/tests/secret-scanner-baseline.test.ts": 1192, - "packages/core/tests/secret-scanner.test.ts": 1345, - "packages/core/tests/shared-operation-permissions.test.ts": 4058, - "packages/core/tests/slice-component.test.ts": 3848, - "packages/core/tests/source-inspection.test.ts": 2940, - "packages/core/tests/source-position.test.ts": 3160, - "packages/core/tests/state-ownership.test.ts": 3211, - "packages/core/tests/streaming-emission.test.ts": 3163, - "packages/core/tests/switch.test.ts": 3603, - "packages/core/tests/syntax-api.test.ts": 3472, - "packages/core/tests/syntax-catalog.test.ts": 3637, - "packages/core/tests/syntax-component.test.ts": 4367, - "packages/core/tests/syntax-loaded-copy.test.ts": 3430, - "packages/core/tests/temp-dir.test.ts": 5711, - "packages/core/tests/temp-file-compiler.test.ts": 1650, - "packages/core/tests/text-interpolation.test.ts": 3198, - "packages/core/tests/try-content.test.ts": 3424, - "packages/core/tests/typescript-materialization.test.ts": 4455, - "packages/core/tests/unused-in-diff.test.ts": 3265, - "packages/core/tests/validate.test.ts": 3124, - "packages/core/tests/validation-integration.test.ts": 3157, - "packages/core/tests/workflow-component-bundle.test.ts": 3514, - "packages/core/tests/yaml-component.test.ts": 3165, - "packages/durable-streams/tests/context.test.ts": 1251, - "packages/durable-streams/tests/deterministic-id.test.ts": 1301, - "packages/durable-streams/tests/divergence-api.test.ts": 1289, - "packages/durable-streams/tests/divergence.test.ts": 1349, - "packages/durable-streams/tests/durable-each.test.ts": 1318, - "packages/durable-streams/tests/durable-position.test.ts": 1290, - "packages/durable-streams/tests/durable-run.test.ts": 1342, - "packages/durable-streams/tests/ephemeral.test.ts": 1277, - "packages/durable-streams/tests/fail-stop.test.ts": 1329, - "packages/durable-streams/tests/guard-stream.test.ts": 1359, - "packages/durable-streams/tests/live-coordinator.test.ts": 1314, - "packages/durable-streams/tests/parse.test.ts": 1200, - "packages/durable-streams/tests/replay-guard.test.ts": 1264, - "packages/durable-streams/tests/replay-index.test.ts": 1051, - "packages/durable-streams/tests/retained.test.ts": 1038, - "packages/durable-streams/tests/serialize-event.test.ts": 1189, - "packages/durable-streams/tests/smoke.test.ts": 1143, - "packages/durable-streams/tests/structured-concurrency.test.ts": 1214, - "packages/durable-streams/tests/terminal-boundary.test.ts": 1204, - "packages/durable-streams/tests/types.test.ts": 962, - "packages/git/tests/ambient-authentication.test.ts": 20959, - "packages/git/tests/api-entrypoint.test.ts": 3800, - "packages/git/tests/composition-component-binding.test.ts": 3571, - "packages/git/tests/credential-helper.test.ts": 4220, - "packages/git/tests/git-add-crash.test.ts": 4942, - "packages/git/tests/git-add-durability.test.ts": 19719, - "packages/git/tests/git-add.test.ts": 27411, - "packages/git/tests/git-commit-crash.test.ts": 4993, - "packages/git/tests/git-commit-durability.test.ts": 26806, - "packages/git/tests/git-commit-read.test.ts": 10334, - "packages/git/tests/git-commit.test.ts": 31235, - "packages/git/tests/git-host-effect.test.ts": 6514, - "packages/git/tests/git-host-owned.test.ts": 4346, - "packages/git/tests/git-observation-records.test.ts": 3114, - "packages/git/tests/git-observations-deno.test.ts": 4512, - "packages/git/tests/git-observations.test.ts": 16353, - "packages/git/tests/git-push-crash.test.ts": 7029, - "packages/git/tests/git-push-durability.test.ts": 35058, - "packages/git/tests/git-push.test.ts": 41989, - "packages/git/tests/git-switch-crash.test.ts": 4854, - "packages/git/tests/git-switch-durability.test.ts": 16152, - "packages/git/tests/git-switch.test.ts": 18442, - "packages/git/tests/git.test.ts": 3494, - "packages/git/tests/github-activation.test.ts": 4104, - "packages/git/tests/github-issues.test.ts": 3246, - "packages/git/tests/github-pull-requests.test.ts": 3699, - "packages/git/tests/github-workflow-activation.test.ts": 4361, - "packages/git/tests/issue-markdown.test.ts": 7648, - "packages/git/tests/issue-owned.test.ts": 4423, - "packages/git/tests/issue-records.test.ts": 3218, - "packages/git/tests/materialization.test.ts": 3477, - "packages/git/tests/module-partition.test.ts": 3305, - "packages/git/tests/plugin.test.ts": 3476, - "packages/git/tests/provider-neutrality.test.ts": 2255, - "packages/git/tests/public-entrypoint.test.ts": 8545, - "packages/git/tests/pull-request-crash.test.ts": 6695, - "packages/git/tests/pull-request-durability.test.ts": 43258, - "packages/git/tests/pull-request-read.test.ts": 8297, - "packages/git/tests/pull-request-records.test.ts": 3710, - "packages/git/tests/pull-request.test.ts": 73583, - "packages/git/tests/repository-components.test.ts": 9341, - "packages/git/tests/repository-control-plane.test.ts": 8204, - "packages/git/tests/repository-materialization.test.ts": 5972, - "packages/git/tests/repository-replay.test.ts": 12465, - "packages/git/tests/repository-storage.test.ts": 14358, - "packages/git/tests/run-composition-ambient.test.ts": 10444, - "packages/git/tests/run-composition-lazy.test.ts": 4117, - "packages/git/tests/run-composition-managed.test.ts": 11579, - "packages/git/tests/run-composition-remote.test.ts": 12913, - "packages/git/tests/selection-authentication.test.ts": 1162, - "packages/git/tests/worktree-replay.test.ts": 4837, - "packages/runtime/tests/agent-session-coordinator.test.ts": 1857, - "packages/runtime/tests/duration.test.ts": 1005, - "packages/runtime/tests/executable-observer.test.ts": 1321, - "packages/runtime/tests/fetch.test.ts": 1761, - "packages/runtime/tests/files-call.test.ts": 1022, - "packages/runtime/tests/fs.test.ts": 1133, - "packages/runtime/tests/host-files.test.ts": 1898, - "packages/runtime/tests/native-launcher.test.ts": 3331, - "packages/runtime/tests/permissions.test.ts": 1238, - "packages/runtime/tests/process.test.ts": 1152, - "packages/runtime/tests/service.test.ts": 1251, - "packages/test-agent/tests/acp-server.test.ts": 1087, - "packages/test-agent/tests/behavior-engine.test.ts": 3316, - "packages/test-agent/tests/bridge.test.ts": 2933, - "packages/test-agent/tests/command-lazy.test.ts": 6901, - "packages/test-agent/tests/components.test.ts": 73274, - "packages/test-agent/tests/controller.test.ts": 3085, - "packages/test-agent/tests/cross-package-resolution.test.ts": 3908, - "packages/test-agent/tests/native-launch.test.ts": 38573, - "packages/test-agent/tests/net.test.ts": 1024, - "packages/test-agent/tests/profile.test.ts": 2916, - "packages/test-agent/tests/protocol.test.ts": 2832, - "packages/test-agent/tests/provider.test.ts": 3170, - "packages/test-agent/tests/public-api.test.ts": 3429, - "packages/test-agent/tests/route-slot.test.ts": 1026, - "packages/test-agent/tests/session-configuration.test.ts": 111916, - "packages/test-agent/tests/smoke.test.ts": 21092, - "packages/test-agent/tests/template.test.ts": 2826, - "packages/test-agent/tests/worker-lifecycle.test.ts": 13321, - "packages/test-support/tests/journal.test.ts": 1140, - "packages/testing/tests/agent-composition.test.ts": 3087, - "packages/testing/tests/assert-throws.test.ts": 3893, - "packages/testing/tests/assert.test.ts": 966, - "packages/testing/tests/assertions.test.ts": 4328, - "packages/testing/tests/boundary-contract.test.ts": 3581, - "packages/testing/tests/cli.test.ts": 6022, - "packages/testing/tests/execution-harness.test.ts": 6085, - "packages/testing/tests/replay.test.ts": 3873, - "packages/testing/tests/smoke.test.ts": 4561, - "packages/testing/tests/test-component.test.ts": 4388, - "packages/testing/tests/testing-activation.test.ts": 3980, - "packages/testing/tests/testing-boundary.test.ts": 3364, - "packages/testing/tests/testing-mode.test.ts": 4399, - "packages/testing/tests/use-testing.test.ts": 3700, - "packages/web/tests/assets.test.ts": 1003, - "packages/web/tests/client-logic.test.ts": 1070, - "packages/web/tests/compile.test.ts": 3302, - "packages/web/tests/component.test.ts": 3885, - "packages/web/tests/declaration.test.ts": 3051, - "packages/web/tests/document.test.ts": 3352, - "packages/web/tests/elicitation.test.ts": 3150, - "packages/web/tests/live-form.test.ts": 3028, - "packages/web/tests/markdown.test.ts": 1093, - "packages/web/tests/opener.test.ts": 3123, - "packages/web/tests/page.test.ts": 1015, - "packages/web/tests/responder.test.ts": 3259, - "packages/web/tests/server-lifecycle.test.ts": 3212, - "packages/web/tests/server.test.ts": 3359, - "packages/workflow/tests/generated-agent-component.test.ts": 4726, - "packages/workflow/tests/generated-observations.test.ts": 3145, - "packages/workflow/tests/public-entrypoint.test.ts": 4059, - "packages/workflow/tests/retained-run.test.ts": 3540, - "packages/workflow/tests/service-denial.test.ts": 1195, - "packages/workflow/tests/workflow-agent-checkpoints.test.ts": 4093, - "packages/workflow/tests/workflow-agent-sessions.test.ts": 3511, - "packages/workflow/tests/workflow-bundle.test.ts": 3176, - "packages/workflow/tests/workflow-checkpoint.test.ts": 4136, - "packages/workflow/tests/workflow-definition.test.ts": 2997, - "packages/workflow/tests/workflow-export.test.ts": 5492, - "packages/workflow/tests/workflow-fork-source.test.ts": 3459, - "packages/workflow/tests/workflow-fork.test.ts": 2895, - "packages/workflow/tests/workflow-lifecycle-authority.test.ts": 4044, - "packages/workflow/tests/workflow-lifecycle-control.test.ts": 4497, - "packages/workflow/tests/workflow-lifecycle-inspection.test.ts": 8545, - "packages/workflow/tests/workflow-run-journal.test.ts": 4885, - "packages/workflow/tests/workflow-run-storage.test.ts": 6006, - "packages/workflow/tests/workflow-run.test.ts": 3462, - "packages/workflow/tests/workflow-suspension-answer.test.ts": 4278, - "packages/workflow/tests/workflow-suspension.test.ts": 4117, - "packages/workflow/tests/workspace-crash-recovery.test.ts": 5309, - "packages/workflow/tests/workspace-effect-loaded-copy.test.ts": 3697, - "packages/workflow/tests/workspace-effect-transaction.test.ts": 4073, - "packages/workflow/tests/workspace-effect.test.ts": 4099, - "packages/workflow/tests/workspace-files.test.ts": 9672, - "packages/workflow/tests/workspace-root-restoration.test.ts": 4687, - "packages/workflow/tests/workspace-root.test.ts": 3525, - "packages/workflow/tests/workspace-transaction.test.ts": 3425, - "packages/workflow/tests/xmd-artifact.test.ts": 4628, - "scripts/tests/acpx-vendor.test.ts": 1440, - "scripts/tests/adapter-distribution.test.ts": 14311, - "scripts/tests/adapter-npm-package.test.ts": 26049, - "scripts/tests/adapter-vendor.test.ts": 3068, - "scripts/tests/bootstrap-npm-package.test.ts": 9278, - "scripts/tests/build-npm.test.ts": 38799, - "scripts/tests/build-web-client.test.ts": 21999, - "scripts/tests/bump-version.test.ts": 1040, - "scripts/tests/changed-paths.test.ts": 940, - "scripts/tests/ci-workflow.test.ts": 3781, - "scripts/tests/claude-preparation.test.ts": 1946, - "scripts/tests/claude-snapshot-module.test.ts": 1269, - "scripts/tests/cli-npm-bin.test.ts": 187460, - "scripts/tests/cloudflare-dofs-vendor.test.ts": 2711, - "scripts/tests/consumer-cycle.test.ts": 1053, - "scripts/tests/documentation-validation.test.ts": 4036, - "scripts/tests/filesystem-contract-workflow.test.ts": 1099, - "scripts/tests/frozen-entry.test.ts": 7860, - "scripts/tests/jsr-consumer-documentation.test.ts": 12601, - "scripts/tests/main-green.test.ts": 1331, - "scripts/tests/main-health.test.ts": 1088, - "scripts/tests/measure-test-weights.test.ts": 1738, - "scripts/tests/no-module-scoped-registry.test.ts": 1570, - "scripts/tests/no-redundant-test-scope.test.ts": 4442, - "scripts/tests/no-sync-filesystem.test.ts": 4415, - "scripts/tests/no-yield-in-finally.test.ts": 1516, - "scripts/tests/oxlint-policy.test.ts": 10293, - "scripts/tests/packaged-document.test.ts": 1021, - "scripts/tests/prefer-effection-operation.test.ts": 8289, - "scripts/tests/prefer-effection-result.test.ts": 3854, - "scripts/tests/prepared-state.test.ts": 1663, - "scripts/tests/publish-workflow-generator.test.ts": 4118, - "scripts/tests/publish-workflow-membership.test.ts": 1062, - "scripts/tests/publishable-membership-agreement.test.ts": 2894, - "scripts/tests/query-distribution.test.ts": 55024, - "scripts/tests/readme-targets.test.ts": 13248, - "scripts/tests/release-targets.test.ts": 1042, - "scripts/tests/review-infrastructure.test.ts": 4855, - "scripts/tests/runtime-exclusions.test.ts": 1338, - "scripts/tests/runtime-tests.test.ts": 1095, - "scripts/tests/scope-bound-event-registration.test.ts": 4291, - "scripts/tests/shard-execution.test.ts": 8920, - "scripts/tests/staged-write.test.ts": 1058, - "scripts/tests/test-file-discovery.test.ts": 1102, - "scripts/tests/test-shards.test.ts": 1003, - "scripts/tests/test-weights.test.ts": 1372, - "scripts/tests/tracked.test.ts": 1009, - "scripts/tests/verify-adapter.test.ts": 6541, - "scripts/tests/verify-clean.test.ts": 1142, - "scripts/tests/verify-coordinator.test.ts": 1307, - "scripts/tests/version-lockstep.test.ts": 1111, - "scripts/tests/web-client-module.test.ts": 971, - "scripts/tests/workspace.test.ts": 986 + "packages/acp/tests/acceptance.test.ts": 5052, + "packages/acp/tests/acpx-checkpoint-meta.test.ts": 10991, + "packages/acp/tests/acpx-materialization.test.ts": 30639, + "packages/acp/tests/acpx-session-save.test.ts": 4498, + "packages/acp/tests/acpx-transient-env.test.ts": 4037, + "packages/acp/tests/adapter-materialization.test.ts": 83281, + "packages/acp/tests/adapter-protocol.test.ts": 51388, + "packages/acp/tests/agent-certification-report.test.ts": 3698, + "packages/acp/tests/agent-certification.test.ts": 41952, + "packages/acp/tests/codex-surfaces.test.ts": 1460, + "packages/acp/tests/host-conversation.test.ts": 12854, + "packages/acp/tests/loaded-copy.test.ts": 7900, + "packages/acp/tests/native-launch.test.ts": 5024, + "packages/acp/tests/permission-bridge.test.ts": 3941, + "packages/acp/tests/prompt-usage.test.ts": 12275, + "packages/acp/tests/provider.test.ts": 5079, + "packages/acp/tests/released-runtime.test.ts": 53094, + "packages/acp/tests/serial-queue.test.ts": 1315, + "packages/acp/tests/session-key.test.ts": 4087, + "packages/acp/tests/session-route.test.ts": 4238, + "packages/acp/tests/terminal-screen.test.ts": 1470, + "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 17851, + "packages/cli/tests/adversarial-artifact-return.test.ts": 37320, + "packages/cli/tests/adversarial-delivery.test.ts": 38386, + "packages/cli/tests/adversarial-friction.test.ts": 110981, + "packages/cli/tests/adversarial-implementation.test.ts": 101511, + "packages/cli/tests/adversarial-launch-policy.test.ts": 74909, + "packages/cli/tests/adversarial-preparation.test.ts": 159796, + "packages/cli/tests/adversarial-production-role.test.ts": 64748, + "packages/cli/tests/adversarial-session-continuity.test.ts": 20332, + "packages/cli/tests/adversarial-workflow.test.ts": 78900, + "packages/cli/tests/agent-adapters.test.ts": 6377, + "packages/cli/tests/agent-certification.test.ts": 223640, + "packages/cli/tests/agent-cli.test.ts": 42736, + "packages/cli/tests/agent-config.test.ts": 3558, + "packages/cli/tests/agent-options-cli.test.ts": 24411, + "packages/cli/tests/agent-options.test.ts": 3573, + "packages/cli/tests/agent-session-coordinator.test.ts": 4411, + "packages/cli/tests/agent-usage-calibration.test.ts": 56966, + "packages/cli/tests/cli-help.test.ts": 52151, + "packages/cli/tests/command.test.ts": 14454, + "packages/cli/tests/compiled-upgrade.test.ts": 11352, + "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 9406, + "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 150452, + "packages/cli/tests/document-suites/props/props-markdown.test.ts": 9033, + "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 12417, + "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 10841, + "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 15718, + "packages/cli/tests/evaluate-component.test.ts": 11131, + "packages/cli/tests/evaluate-workflow.test.ts": 5265, + "packages/cli/tests/fetch-cli.test.ts": 3543, + "packages/cli/tests/file-stream.test.ts": 1467, + "packages/cli/tests/github-zero-effect.test.ts": 6771, + "packages/cli/tests/inline-cli.test.ts": 31539, + "packages/cli/tests/launch-timeout.test.ts": 11399, + "packages/cli/tests/packaged-document.test.ts": 4608, + "packages/cli/tests/parse-failure.test.ts": 4381, + "packages/cli/tests/plan-args.test.ts": 1270, + "packages/cli/tests/plan-cli.test.ts": 100784, + "packages/cli/tests/plan-command-document.test.ts": 9766, + "packages/cli/tests/plan-component.test.ts": 11511, + "packages/cli/tests/plan-host-acts.test.ts": 5052, + "packages/cli/tests/plan.test.ts": 14439, + "packages/cli/tests/plugin-cli.test.ts": 39157, + "packages/cli/tests/plugin-host.test.ts": 5912, + "packages/cli/tests/plugin-modules.test.ts": 7752, + "packages/cli/tests/plugin-selection.test.ts": 1322, + "packages/cli/tests/process-retention.test.ts": 6033, + "packages/cli/tests/prompt-usage-hosts.test.ts": 8385, + "packages/cli/tests/props-cli.test.ts": 29963, + "packages/cli/tests/props-schema.test.ts": 4029, + "packages/cli/tests/props-sources.test.ts": 2005, + "packages/cli/tests/protected-input-cli.test.ts": 10711, + "packages/cli/tests/protected-input-forms.test.ts": 7531, + "packages/cli/tests/pull-request-read-fork.test.ts": 7623, + "packages/cli/tests/repl-admission.test.ts": 1367, + "packages/cli/tests/repl-agent-execution.test.ts": 7811, + "packages/cli/tests/repl-agent-interface.test.ts": 35187, + "packages/cli/tests/repl-agent-journey.test.ts": 24556, + "packages/cli/tests/repl-asset-import.test.ts": 12466, + "packages/cli/tests/repl-boundaries.test.ts": 8503, + "packages/cli/tests/repl-caret.test.ts": 7033, + "packages/cli/tests/repl-composition.test.ts": 9509, + "packages/cli/tests/repl-entries.test.ts": 13908, + "packages/cli/tests/repl-exec-output.test.ts": 11066, + "packages/cli/tests/repl-execution.test.ts": 6339, + "packages/cli/tests/repl-exit-pty.test.ts": 6761, + "packages/cli/tests/repl-field-ownership.test.ts": 7281, + "packages/cli/tests/repl-focus-navigation.test.ts": 1612, + "packages/cli/tests/repl-focus-regions.test.ts": 16130, + "packages/cli/tests/repl-forms.test.ts": 11138, + "packages/cli/tests/repl-git-observations.test.ts": 8292, + "packages/cli/tests/repl-headless-attach.test.ts": 54834, + "packages/cli/tests/repl-headless-clients.test.ts": 63173, + "packages/cli/tests/repl-headless-submit-process.test.ts": 20514, + "packages/cli/tests/repl-headless-submit.test.ts": 29042, + "packages/cli/tests/repl-headless.test.ts": 17254, + "packages/cli/tests/repl-history.test.ts": 6228, + "packages/cli/tests/repl-include-pty.test.ts": 28060, + "packages/cli/tests/repl-issues-pty.test.ts": 11688, + "packages/cli/tests/repl-issues.test.ts": 12513, + "packages/cli/tests/repl-journey.test.ts": 41543, + "packages/cli/tests/repl-layout-admission.test.ts": 12321, + "packages/cli/tests/repl-lifecycle.test.ts": 4877, + "packages/cli/tests/repl-model.test.ts": 5297, + "packages/cli/tests/repl-mounts-pty.test.ts": 27663, + "packages/cli/tests/repl-page.test.ts": 9679, + "packages/cli/tests/repl-parse-failure.test.ts": 4604, + "packages/cli/tests/repl-plugin.test.ts": 8517, + "packages/cli/tests/repl-presentation-style.test.ts": 37221, + "packages/cli/tests/repl-project-files.test.ts": 18064, + "packages/cli/tests/repl-project-recovery.test.ts": 13092, + "packages/cli/tests/repl-push-pty.test.ts": 23498, + "packages/cli/tests/repl-push.test.ts": 11318, + "packages/cli/tests/repl-repositories-pty.test.ts": 45098, + "packages/cli/tests/repl-repositories.test.ts": 12091, + "packages/cli/tests/repl-retained-history.test.ts": 7466, + "packages/cli/tests/repl-route.test.ts": 7608, + "packages/cli/tests/repl-sidekick-acp.test.ts": 269690, + "packages/cli/tests/repl-sidekick-configuration.test.ts": 26558, + "packages/cli/tests/repl-sidekick-controls.test.ts": 52741, + "packages/cli/tests/repl-sidekick-dispatch.test.ts": 15422, + "packages/cli/tests/repl-sidekick-journey.test.ts": 34600, + "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 22081, + "packages/cli/tests/repl-sidekick-output.test.ts": 1325, + "packages/cli/tests/repl-sidekick-presentation.test.ts": 92217, + "packages/cli/tests/repl-sidekick-recovery.test.ts": 164868, + "packages/cli/tests/repl-sidekick-storage.test.ts": 4381, + "packages/cli/tests/repl-sidekick-testing.test.ts": 25749, + "packages/cli/tests/repl-sidekick-typing.test.ts": 23561, + "packages/cli/tests/repl-source-reading.test.ts": 8775, + "packages/cli/tests/repl-task-tabs.test.ts": 11033, + "packages/cli/tests/repl-terminal.test.ts": 17537, + "packages/cli/tests/repl-todo-context.test.ts": 8074, + "packages/cli/tests/repl-todo-journey.test.ts": 15292, + "packages/cli/tests/repl-todo-recovery.test.ts": 8425, + "packages/cli/tests/repl-todo.test.ts": 4217, + "packages/cli/tests/repl-unfinished-entry.test.ts": 16629, + "packages/cli/tests/repl-wheel-frame.test.ts": 7552, + "packages/cli/tests/repl-wheel-input.test.ts": 7019, + "packages/cli/tests/repl-wheel-migration.test.ts": 40617, + "packages/cli/tests/repl-wheel-scroll.test.ts": 36082, + "packages/cli/tests/retained-mount-files.test.ts": 5625, + "packages/cli/tests/retained-mount-records.test.ts": 5185, + "packages/cli/tests/retained-repository-git.test.ts": 16198, + "packages/cli/tests/retained-repository-history.test.ts": 7140, + "packages/cli/tests/retained-repository-lifecycle.test.ts": 11748, + "packages/cli/tests/retained-repository-local.test.ts": 18250, + "packages/cli/tests/retained-repository-observations-deno.test.ts": 8941, + "packages/cli/tests/retained-repository-observations.test.ts": 10215, + "packages/cli/tests/retained-repository-pull-request-records.test.ts": 5018, + "packages/cli/tests/retained-repository-pull-request.test.ts": 26202, + "packages/cli/tests/retained-repository-push-records.test.ts": 5171, + "packages/cli/tests/retained-repository-push.test.ts": 25946, + "packages/cli/tests/retained-repository-records.test.ts": 4339, + "packages/cli/tests/retained-repository-selection.test.ts": 29556, + "packages/cli/tests/run-composition-deno.test.ts": 8231, + "packages/cli/tests/run-composition-nested.test.ts": 22922, + "packages/cli/tests/run-composition.test.ts": 8747, + "packages/cli/tests/run-deadline.test.ts": 9714, + "packages/cli/tests/run-profile.test.ts": 4793, + "packages/cli/tests/run-timeouts.test.ts": 12977, + "packages/cli/tests/secret-detection-cli.test.ts": 19411, + "packages/cli/tests/service-document.test.ts": 5302, + "packages/cli/tests/service-host.test.ts": 5129, + "packages/cli/tests/session-launch-cli.test.ts": 11628, + "packages/cli/tests/session-ownership-cli.test.ts": 12634, + "packages/cli/tests/stdin-cli.test.ts": 38380, + "packages/cli/tests/stdout-delivery.test.ts": 1282, + "packages/cli/tests/syntax-cli.test.ts": 34502, + "packages/cli/tests/targets-cli.test.ts": 29105, + "packages/cli/tests/test-root-profile.test.ts": 8066, + "packages/cli/tests/test-target.test.ts": 35428, + "packages/cli/tests/testing-activation.test.ts": 4723, + "packages/cli/tests/testing-execution-host-native.test.ts": 9028, + "packages/cli/tests/testing-execution-host.test.ts": 66932, + "packages/cli/tests/upgrade-cli.test.ts": 20469, + "packages/cli/tests/upgrade-command-document.test.ts": 7081, + "packages/cli/tests/upgrade-output.test.ts": 4452, + "packages/cli/tests/value-root.test.ts": 17011, + "packages/cli/tests/verbose-component.test.ts": 4987, + "packages/cli/tests/workflow-activation.test.ts": 65159, + "packages/cli/tests/workflow-agent-sessions.test.ts": 37131, + "packages/cli/tests/workflow-agent.test.ts": 15035, + "packages/cli/tests/workflow-cli.test.ts": 120478, + "packages/cli/tests/workflow-crash.test.ts": 83022, + "packages/cli/tests/workflow-declaration.test.ts": 4471, + "packages/cli/tests/workflow-export-publication.test.ts": 19892, + "packages/cli/tests/workflow-fetch.test.ts": 8290, + "packages/cli/tests/workflow-fork.test.ts": 87062, + "packages/cli/tests/workflow-host.test.ts": 14463, + "packages/cli/tests/workflow-inspection.test.ts": 120413, + "packages/cli/tests/workflow-installation.test.ts": 5918, + "packages/cli/tests/workflow-lifecycle-control.test.ts": 20789, + "packages/cli/tests/workflow-retention.test.ts": 23869, + "packages/cli/tests/workflow-suspension.test.ts": 12373, + "packages/cli/tests/workspace-mounts.test.ts": 11631, + "packages/code-review-agent/tests/doctor.test.ts": 1282, + "packages/code-review-agent/tests/parse-diagnostics.test.ts": 1330, + "packages/code-review-agent/tests/parse-diff.test.ts": 1277, + "packages/code-review-agent/tests/parse-doctor.test.ts": 4046, + "packages/code-review-agent/tests/parse-oxlint.test.ts": 1316, + "packages/code-review-agent/tests/policy.test.ts": 1333, + "packages/core/tests/agent-api.test.ts": 3608, + "packages/core/tests/agent-components.test.ts": 4672, + "packages/core/tests/agent-conversation.test.ts": 7040, + "packages/core/tests/agent-function-components.test.ts": 6717, + "packages/core/tests/agent-session-launch.test.ts": 6264, + "packages/core/tests/agent-session-setup.test.ts": 3544, + "packages/core/tests/agent-session-use-loaded-copy.test.ts": 3657, + "packages/core/tests/agent-session-use.test.ts": 3542, + "packages/core/tests/agent-stack.test.ts": 4004, + "packages/core/tests/agent-usage.test.ts": 4173, + "packages/core/tests/all.test.ts": 5567, + "packages/core/tests/answer-identity.test.ts": 3598, + "packages/core/tests/answers-component.test.ts": 4827, + "packages/core/tests/answers-expansion-recursion.test.ts": 3733, + "packages/core/tests/capture-errors.test.ts": 3850, + "packages/core/tests/capture-props.test.ts": 3668, + "packages/core/tests/cli-journal.test.ts": 11142, + "packages/core/tests/code-block-component.test.ts": 4931, + "packages/core/tests/compiler-boundary.test.ts": 3677, + "packages/core/tests/component-api.test.ts": 3482, + "packages/core/tests/component-expansion.test.ts": 3907, + "packages/core/tests/component-invocation-hooks.test.ts": 4450, + "packages/core/tests/component-module-loader.test.ts": 5881, + "packages/core/tests/component-registration.test.ts": 4855, + "packages/core/tests/component-returns.test.ts": 5608, + "packages/core/tests/component-schema-conformance.test.ts": 3741, + "packages/core/tests/config-api.test.ts": 3827, + "packages/core/tests/construct-error-observation.test.ts": 3955, + "packages/core/tests/contextual-cwd.test.ts": 5639, + "packages/core/tests/daemon-integration.test.ts": 4668, + "packages/core/tests/daemon.test.ts": 3551, + "packages/core/tests/declared-markdown-component.test.ts": 5426, + "packages/core/tests/document-output-api.test.ts": 1261, + "packages/core/tests/document-target-execution.test.ts": 6439, + "packages/core/tests/document-targets.test.ts": 3962, + "packages/core/tests/document-validation.test.ts": 5212, + "packages/core/tests/documentation-index.test.ts": 3577, + "packages/core/tests/each.test.ts": 3842, + "packages/core/tests/elicit-component.test.ts": 4631, + "packages/core/tests/ephemeral-service.test.ts": 4355, + "packages/core/tests/eval-bindings.test.ts": 4151, + "packages/core/tests/eval-context.test.ts": 2032, + "packages/core/tests/eval-durable.test.ts": 4024, + "packages/core/tests/eval-error-mode.test.ts": 4218, + "packages/core/tests/eval-interpolate.test.ts": 3494, + "packages/core/tests/eval-middleware.test.ts": 3611, + "packages/core/tests/eval-persist.test.ts": 4118, + "packages/core/tests/eval-return.test.ts": 3938, + "packages/core/tests/eval-scope.test.ts": 3838, + "packages/core/tests/eval-timeout.test.ts": 3798, + "packages/core/tests/eval-transform.test.ts": 3575, + "packages/core/tests/evaluate-component.test.ts": 6113, + "packages/core/tests/evaluate-loaded-copy.test.ts": 4031, + "packages/core/tests/evaluate-provider-lifetime.test.ts": 4027, + "packages/core/tests/evaluation-profile.test.ts": 3641, + "packages/core/tests/exec-timeout.test.ts": 3726, + "packages/core/tests/execute.test.ts": 4365, + "packages/core/tests/execution-init.test.ts": 3902, + "packages/core/tests/execution-protocol.test.ts": 5023, + "packages/core/tests/expand.test.ts": 4802, + "packages/core/tests/expansion-identity.test.ts": 4240, + "packages/core/tests/expansion-metadata.test.ts": 3582, + "packages/core/tests/expression-props.test.ts": 4120, + "packages/core/tests/fail-component.test.ts": 4045, + "packages/core/tests/failure-printing.test.ts": 3907, + "packages/core/tests/fatal-cause.test.ts": 3608, + "packages/core/tests/fetch-component.test.ts": 5207, + "packages/core/tests/file-component.test.ts": 5638, + "packages/core/tests/file-delete-component.test.ts": 3984, + "packages/core/tests/files-fatal.test.ts": 4508, + "packages/core/tests/filesystem-placement.test.ts": 4557, + "packages/core/tests/fixed-filesystem-components.test.ts": 4116, + "packages/core/tests/foreground-exec.test.ts": 4742, + "packages/core/tests/frontmatter.test.ts": 3505, + "packages/core/tests/function-components.test.ts": 4756, + "packages/core/tests/generated-composition.test.ts": 4458, + "packages/core/tests/generated-owner-metadata.test.ts": 3930, + "packages/core/tests/generated-selection.test.ts": 4534, + "packages/core/tests/generated-xmd-preview.test.ts": 3804, + "packages/core/tests/generated-xmd-root.test.ts": 5002, + "packages/core/tests/generated-xmd.test.ts": 5506, + "packages/core/tests/glob-api.test.ts": 1637, + "packages/core/tests/glob-component.test.ts": 4597, + "packages/core/tests/guarded-journal.test.ts": 3956, + "packages/core/tests/has-content.test.ts": 3683, + "packages/core/tests/heal.test.ts": 3780, + "packages/core/tests/host-generated-root.test.ts": 4816, + "packages/core/tests/if.test.ts": 4433, + "packages/core/tests/inline-root.test.ts": 4017, + "packages/core/tests/invocation-failures.test.ts": 3713, + "packages/core/tests/invocation-identity.test.ts": 5080, + "packages/core/tests/invocation-scope.test.ts": 4023, + "packages/core/tests/journal-source-position.test.ts": 4076, + "packages/core/tests/json-component.test.ts": 4305, + "packages/core/tests/json.test.ts": 3567, + "packages/core/tests/let.test.ts": 4222, + "packages/core/tests/loaded-copy-files.test.ts": 3877, + "packages/core/tests/loop.test.ts": 4577, + "packages/core/tests/named-slots.test.ts": 4430, + "packages/core/tests/output-error-mode.test.ts": 4805, + "packages/core/tests/output-normalize.test.ts": 1251, + "packages/core/tests/output-terminal.test.ts": 1356, + "packages/core/tests/parse-components.test.ts": 4006, + "packages/core/tests/plan-response.test.ts": 1294, + "packages/core/tests/plugin-api.test.ts": 3600, + "packages/core/tests/plugin.test.ts": 3694, + "packages/core/tests/props-binding.test.ts": 3896, + "packages/core/tests/protected-content.test.ts": 3484, + "packages/core/tests/protected-input-replay.test.ts": 4185, + "packages/core/tests/protected-input.test.ts": 4788, + "packages/core/tests/query-component.test.ts": 4043, + "packages/core/tests/registered-printed-errors.test.ts": 3563, + "packages/core/tests/replay-stream.test.ts": 1282, + "packages/core/tests/retain.test.ts": 3750, + "packages/core/tests/root-composition.test.ts": 3850, + "packages/core/tests/root-props.test.ts": 3844, + "packages/core/tests/root-provider.test.ts": 3835, + "packages/core/tests/sample-component.test.ts": 4515, + "packages/core/tests/scanner.test.ts": 3585, + "packages/core/tests/scope-local.test.ts": 1276, + "packages/core/tests/secret-detection.test.ts": 4488, + "packages/core/tests/secret-files.test.ts": 1733, + "packages/core/tests/secret-rules.test.ts": 1822, + "packages/core/tests/secret-scanner-baseline.test.ts": 1410, + "packages/core/tests/secret-scanner.test.ts": 1546, + "packages/core/tests/shared-operation-permissions.test.ts": 4722, + "packages/core/tests/slice-component.test.ts": 4358, + "packages/core/tests/source-inspection.test.ts": 3599, + "packages/core/tests/source-position.test.ts": 3680, + "packages/core/tests/state-ownership.test.ts": 3672, + "packages/core/tests/streaming-emission.test.ts": 3715, + "packages/core/tests/switch.test.ts": 4410, + "packages/core/tests/syntax-api.test.ts": 4728, + "packages/core/tests/syntax-catalog.test.ts": 4655, + "packages/core/tests/syntax-component.test.ts": 4947, + "packages/core/tests/syntax-loaded-copy.test.ts": 4054, + "packages/core/tests/temp-dir.test.ts": 6261, + "packages/core/tests/temp-file-compiler.test.ts": 1915, + "packages/core/tests/text-interpolation.test.ts": 3681, + "packages/core/tests/try-content.test.ts": 3755, + "packages/core/tests/typescript-materialization.test.ts": 5189, + "packages/core/tests/unused-in-diff.test.ts": 3905, + "packages/core/tests/validate.test.ts": 3722, + "packages/core/tests/validation-integration.test.ts": 3733, + "packages/core/tests/workflow-component-bundle.test.ts": 4236, + "packages/core/tests/yaml-component.test.ts": 3616, + "packages/durable-streams/tests/context.test.ts": 1469, + "packages/durable-streams/tests/deterministic-id.test.ts": 1524, + "packages/durable-streams/tests/divergence-api.test.ts": 1481, + "packages/durable-streams/tests/divergence.test.ts": 1526, + "packages/durable-streams/tests/durable-each.test.ts": 1581, + "packages/durable-streams/tests/durable-position.test.ts": 1493, + "packages/durable-streams/tests/durable-run.test.ts": 1574, + "packages/durable-streams/tests/ephemeral.test.ts": 1505, + "packages/durable-streams/tests/fail-stop.test.ts": 1554, + "packages/durable-streams/tests/guard-stream.test.ts": 1592, + "packages/durable-streams/tests/live-coordinator.test.ts": 1614, + "packages/durable-streams/tests/parse.test.ts": 1542, + "packages/durable-streams/tests/replay-guard.test.ts": 1582, + "packages/durable-streams/tests/replay-index.test.ts": 1331, + "packages/durable-streams/tests/retained.test.ts": 1399, + "packages/durable-streams/tests/serialize-event.test.ts": 1569, + "packages/durable-streams/tests/smoke.test.ts": 1476, + "packages/durable-streams/tests/structured-concurrency.test.ts": 1629, + "packages/durable-streams/tests/terminal-boundary.test.ts": 1511, + "packages/durable-streams/tests/types.test.ts": 1246, + "packages/git/tests/ambient-authentication.test.ts": 29350, + "packages/git/tests/api-entrypoint.test.ts": 4825, + "packages/git/tests/composition-component-binding.test.ts": 4667, + "packages/git/tests/credential-helper.test.ts": 5548, + "packages/git/tests/git-add-crash.test.ts": 6275, + "packages/git/tests/git-add-durability.test.ts": 29455, + "packages/git/tests/git-add.test.ts": 41636, + "packages/git/tests/git-commit-crash.test.ts": 6971, + "packages/git/tests/git-commit-durability.test.ts": 47058, + "packages/git/tests/git-commit-read.test.ts": 12976, + "packages/git/tests/git-commit.test.ts": 43741, + "packages/git/tests/git-host-effect.test.ts": 8924, + "packages/git/tests/git-host-owned.test.ts": 6263, + "packages/git/tests/git-observation-records.test.ts": 4294, + "packages/git/tests/git-observations-deno.test.ts": 6578, + "packages/git/tests/git-observations.test.ts": 25526, + "packages/git/tests/git-push-crash.test.ts": 10113, + "packages/git/tests/git-push-durability.test.ts": 59169, + "packages/git/tests/git-push.test.ts": 62973, + "packages/git/tests/git-switch-crash.test.ts": 6833, + "packages/git/tests/git-switch-durability.test.ts": 23795, + "packages/git/tests/git-switch.test.ts": 25811, + "packages/git/tests/git.test.ts": 4644, + "packages/git/tests/github-activation.test.ts": 5449, + "packages/git/tests/github-issues.test.ts": 4400, + "packages/git/tests/github-pull-requests.test.ts": 4971, + "packages/git/tests/github-workflow-activation.test.ts": 5939, + "packages/git/tests/issue-markdown.test.ts": 11095, + "packages/git/tests/issue-owned.test.ts": 6161, + "packages/git/tests/issue-records.test.ts": 4297, + "packages/git/tests/materialization.test.ts": 4816, + "packages/git/tests/module-partition.test.ts": 4488, + "packages/git/tests/plugin.test.ts": 4722, + "packages/git/tests/provider-neutrality.test.ts": 3097, + "packages/git/tests/public-entrypoint.test.ts": 10706, + "packages/git/tests/pull-request-crash.test.ts": 7932, + "packages/git/tests/pull-request-durability.test.ts": 49113, + "packages/git/tests/pull-request-read.test.ts": 10103, + "packages/git/tests/pull-request-records.test.ts": 4201, + "packages/git/tests/pull-request.test.ts": 91077, + "packages/git/tests/repository-components.test.ts": 15109, + "packages/git/tests/repository-control-plane.test.ts": 13655, + "packages/git/tests/repository-materialization.test.ts": 9010, + "packages/git/tests/repository-replay.test.ts": 18964, + "packages/git/tests/repository-storage.test.ts": 23037, + "packages/git/tests/run-composition-ambient.test.ts": 14524, + "packages/git/tests/run-composition-lazy.test.ts": 5722, + "packages/git/tests/run-composition-managed.test.ts": 15807, + "packages/git/tests/run-composition-remote.test.ts": 17556, + "packages/git/tests/selection-authentication.test.ts": 1530, + "packages/git/tests/worktree-replay.test.ts": 6944, + "packages/runtime/tests/agent-session-coordinator.test.ts": 2665, + "packages/runtime/tests/duration.test.ts": 1320, + "packages/runtime/tests/executable-observer.test.ts": 1656, + "packages/runtime/tests/fetch.test.ts": 2143, + "packages/runtime/tests/files-call.test.ts": 1417, + "packages/runtime/tests/fs.test.ts": 1547, + "packages/runtime/tests/host-files.test.ts": 2533, + "packages/runtime/tests/native-launcher.test.ts": 3727, + "packages/runtime/tests/permissions.test.ts": 1675, + "packages/runtime/tests/process.test.ts": 1544, + "packages/runtime/tests/service.test.ts": 1641, + "packages/test-agent/tests/acp-server.test.ts": 1450, + "packages/test-agent/tests/behavior-engine.test.ts": 4667, + "packages/test-agent/tests/bridge.test.ts": 3927, + "packages/test-agent/tests/command-lazy.test.ts": 8970, + "packages/test-agent/tests/components.test.ts": 95115, + "packages/test-agent/tests/controller.test.ts": 4338, + "packages/test-agent/tests/cross-package-resolution.test.ts": 5980, + "packages/test-agent/tests/native-launch.test.ts": 51131, + "packages/test-agent/tests/net.test.ts": 1605, + "packages/test-agent/tests/profile.test.ts": 4035, + "packages/test-agent/tests/protocol.test.ts": 3979, + "packages/test-agent/tests/provider.test.ts": 4199, + "packages/test-agent/tests/public-api.test.ts": 4769, + "packages/test-agent/tests/route-slot.test.ts": 1335, + "packages/test-agent/tests/session-configuration.test.ts": 142175, + "packages/test-agent/tests/smoke.test.ts": 26721, + "packages/test-agent/tests/template.test.ts": 3915, + "packages/test-agent/tests/worker-lifecycle.test.ts": 17688, + "packages/test-support/tests/journal.test.ts": 1555, + "packages/testing/tests/agent-composition.test.ts": 4293, + "packages/testing/tests/assert-throws.test.ts": 5272, + "packages/testing/tests/assert.test.ts": 1338, + "packages/testing/tests/assertions.test.ts": 5806, + "packages/testing/tests/boundary-contract.test.ts": 5020, + "packages/testing/tests/cli.test.ts": 8242, + "packages/testing/tests/execution-harness.test.ts": 8695, + "packages/testing/tests/replay.test.ts": 5304, + "packages/testing/tests/smoke.test.ts": 5961, + "packages/testing/tests/test-component.test.ts": 5923, + "packages/testing/tests/testing-activation.test.ts": 5399, + "packages/testing/tests/testing-boundary.test.ts": 4654, + "packages/testing/tests/testing-mode.test.ts": 5937, + "packages/testing/tests/use-testing.test.ts": 4949, + "packages/web/tests/assets.test.ts": 1356, + "packages/web/tests/client-logic.test.ts": 1468, + "packages/web/tests/compile.test.ts": 4430, + "packages/web/tests/component.test.ts": 5581, + "packages/web/tests/declaration.test.ts": 4328, + "packages/web/tests/document.test.ts": 4496, + "packages/web/tests/elicitation.test.ts": 4418, + "packages/web/tests/live-form.test.ts": 4073, + "packages/web/tests/markdown.test.ts": 1522, + "packages/web/tests/opener.test.ts": 4184, + "packages/web/tests/page.test.ts": 1356, + "packages/web/tests/responder.test.ts": 4310, + "packages/web/tests/server-lifecycle.test.ts": 4392, + "packages/web/tests/server.test.ts": 4616, + "packages/workflow/tests/generated-agent-component.test.ts": 6486, + "packages/workflow/tests/generated-observations.test.ts": 4388, + "packages/workflow/tests/public-entrypoint.test.ts": 5482, + "packages/workflow/tests/retained-run.test.ts": 4931, + "packages/workflow/tests/service-denial.test.ts": 1607, + "packages/workflow/tests/workflow-agent-checkpoints.test.ts": 6717, + "packages/workflow/tests/workflow-agent-sessions.test.ts": 4577, + "packages/workflow/tests/workflow-bundle.test.ts": 4311, + "packages/workflow/tests/workflow-checkpoint.test.ts": 5760, + "packages/workflow/tests/workflow-definition.test.ts": 4141, + "packages/workflow/tests/workflow-export.test.ts": 7378, + "packages/workflow/tests/workflow-fork-source.test.ts": 4638, + "packages/workflow/tests/workflow-fork.test.ts": 3917, + "packages/workflow/tests/workflow-lifecycle-authority.test.ts": 5380, + "packages/workflow/tests/workflow-lifecycle-control.test.ts": 5955, + "packages/workflow/tests/workflow-lifecycle-inspection.test.ts": 11113, + "packages/workflow/tests/workflow-run-journal.test.ts": 6761, + "packages/workflow/tests/workflow-run-storage.test.ts": 8461, + "packages/workflow/tests/workflow-run.test.ts": 4851, + "packages/workflow/tests/workflow-suspension-answer.test.ts": 6345, + "packages/workflow/tests/workflow-suspension.test.ts": 5800, + "packages/workflow/tests/workspace-crash-recovery.test.ts": 7159, + "packages/workflow/tests/workspace-effect-loaded-copy.test.ts": 4876, + "packages/workflow/tests/workspace-effect-transaction.test.ts": 6210, + "packages/workflow/tests/workspace-effect.test.ts": 5713, + "packages/workflow/tests/workspace-files.test.ts": 18236, + "packages/workflow/tests/workspace-root-restoration.test.ts": 6652, + "packages/workflow/tests/workspace-root.test.ts": 4656, + "packages/workflow/tests/workspace-transaction.test.ts": 4585, + "packages/workflow/tests/xmd-artifact.test.ts": 6538, + "scripts/tests/acpx-vendor.test.ts": 1896, + "scripts/tests/adapter-distribution.test.ts": 19250, + "scripts/tests/adapter-npm-package.test.ts": 33186, + "scripts/tests/adapter-vendor.test.ts": 4083, + "scripts/tests/bootstrap-npm-package.test.ts": 12167, + "scripts/tests/build-npm.test.ts": 50556, + "scripts/tests/build-web-client.test.ts": 31532, + "scripts/tests/bump-version.test.ts": 1498, + "scripts/tests/changed-paths.test.ts": 1265, + "scripts/tests/ci-workflow.test.ts": 4888, + "scripts/tests/claude-preparation.test.ts": 2337, + "scripts/tests/claude-snapshot-module.test.ts": 1650, + "scripts/tests/cli-npm-bin.test.ts": 272259, + "scripts/tests/cloudflare-dofs-vendor.test.ts": 3733, + "scripts/tests/consumer-cycle.test.ts": 1390, + "scripts/tests/documentation-validation.test.ts": 5733, + "scripts/tests/filesystem-contract-workflow.test.ts": 1312, + "scripts/tests/frozen-entry.test.ts": 8792, + "scripts/tests/jsr-consumer-documentation.test.ts": 17416, + "scripts/tests/main-green.test.ts": 1707, + "scripts/tests/main-health.test.ts": 1447, + "scripts/tests/measure-test-weights.test.ts": 2279, + "scripts/tests/no-module-scoped-registry.test.ts": 2117, + "scripts/tests/no-redundant-test-scope.test.ts": 6058, + "scripts/tests/no-sync-filesystem.test.ts": 5953, + "scripts/tests/no-yield-in-finally.test.ts": 2159, + "scripts/tests/oxlint-policy.test.ts": 14077, + "scripts/tests/packaged-document.test.ts": 1372, + "scripts/tests/prefer-effection-operation.test.ts": 10765, + "scripts/tests/prefer-effection-result.test.ts": 5225, + "scripts/tests/prepared-state.test.ts": 2219, + "scripts/tests/publish-workflow-generator.test.ts": 5794, + "scripts/tests/publish-workflow-membership.test.ts": 1474, + "scripts/tests/publishable-membership-agreement.test.ts": 3886, + "scripts/tests/query-distribution.test.ts": 68738, + "scripts/tests/readme-targets.test.ts": 17256, + "scripts/tests/release-targets.test.ts": 1351, + "scripts/tests/review-infrastructure.test.ts": 6390, + "scripts/tests/runtime-exclusions.test.ts": 1774, + "scripts/tests/runtime-tests.test.ts": 1396, + "scripts/tests/scope-bound-event-registration.test.ts": 5682, + "scripts/tests/shard-execution.test.ts": 10341, + "scripts/tests/staged-write.test.ts": 1292, + "scripts/tests/test-file-discovery.test.ts": 1453, + "scripts/tests/test-shards.test.ts": 1289, + "scripts/tests/test-weights.test.ts": 1757, + "scripts/tests/tracked.test.ts": 1300, + "scripts/tests/verify-adapter.test.ts": 7821, + "scripts/tests/verify-clean.test.ts": 1501, + "scripts/tests/verify-coordinator.test.ts": 1659, + "scripts/tests/version-lockstep.test.ts": 1665, + "scripts/tests/web-client-module.test.ts": 1300, + "scripts/tests/workspace.test.ts": 1323 }, "node": { - "packages/acp/tests/acceptance.test.ts": 2608, - "packages/acp/tests/acpx-checkpoint-meta.test.ts": 6599, - "packages/acp/tests/acpx-materialization.test.ts": 21616, - "packages/acp/tests/acpx-session-save.test.ts": 1013, - "packages/acp/tests/acpx-transient-env.test.ts": 2251, - "packages/acp/tests/adapter-materialization.test.ts": 100357, - "packages/acp/tests/adapter-protocol.test.ts": 52070, - "packages/acp/tests/agent-certification-report.test.ts": 587, - "packages/acp/tests/agent-certification.test.ts": 30625, - "packages/acp/tests/codex-surfaces.test.ts": 709, - "packages/acp/tests/host-conversation.test.ts": 8656, - "packages/acp/tests/native-launch.test.ts": 2601, - "packages/acp/tests/permission-bridge.test.ts": 1409, - "packages/acp/tests/provider.test.ts": 2535, - "packages/acp/tests/released-runtime.test.ts": 35993, - "packages/acp/tests/serial-queue.test.ts": 509, - "packages/acp/tests/session-key.test.ts": 978, - "packages/acp/tests/terminal-screen.test.ts": 675, - "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 23037, - "packages/cli/tests/adversarial-artifact-return.test.ts": 46799, - "packages/cli/tests/adversarial-delivery.test.ts": 47165, - "packages/cli/tests/adversarial-friction.test.ts": 132958, - "packages/cli/tests/adversarial-implementation.test.ts": 122007, - "packages/cli/tests/adversarial-launch-policy.test.ts": 90452, - "packages/cli/tests/adversarial-preparation.test.ts": 187168, - "packages/cli/tests/adversarial-production-role.test.ts": 112428, - "packages/cli/tests/adversarial-session-continuity.test.ts": 13420, - "packages/cli/tests/adversarial-workflow.test.ts": 99090, - "packages/cli/tests/agent-adapters.test.ts": 6144, - "packages/cli/tests/agent-certification.test.ts": 247272, - "packages/cli/tests/agent-cli.test.ts": 69935, - "packages/cli/tests/agent-config.test.ts": 534, - "packages/cli/tests/agent-options-cli.test.ts": 30897, - "packages/cli/tests/agent-options.test.ts": 518, - "packages/cli/tests/agent-session-coordinator.test.ts": 1771, - "packages/cli/tests/cli-help.test.ts": 208524, - "packages/cli/tests/command.test.ts": 29607, - "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 5762, - "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 256490, - "packages/cli/tests/document-suites/props/props-markdown.test.ts": 4474, - "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 14947, - "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 7549, - "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 27860, - "packages/cli/tests/evaluate-component.test.ts": 23449, - "packages/cli/tests/fetch-cli.test.ts": 7411, - "packages/cli/tests/file-stream.test.ts": 565, - "packages/cli/tests/inline-cli.test.ts": 107514, - "packages/cli/tests/launch-timeout.test.ts": 20574, - "packages/cli/tests/packaged-document.test.ts": 2038, - "packages/cli/tests/plan-args.test.ts": 547, - "packages/cli/tests/plan-cli.test.ts": 192036, - "packages/cli/tests/plan-command-document.test.ts": 10446, - "packages/cli/tests/plan-component.test.ts": 13855, - "packages/cli/tests/plan-host-acts.test.ts": 2750, - "packages/cli/tests/plan.test.ts": 18985, - "packages/cli/tests/plugin-cli.test.ts": 130685, - "packages/cli/tests/plugin-host.test.ts": 3849, - "packages/cli/tests/plugin-modules.test.ts": 3313, - "packages/cli/tests/plugin-selection.test.ts": 548, - "packages/cli/tests/process-retention.test.ts": 7452, - "packages/cli/tests/props-cli.test.ts": 103134, - "packages/cli/tests/props-schema.test.ts": 1413, - "packages/cli/tests/props-sources.test.ts": 745, - "packages/cli/tests/protected-input-cli.test.ts": 23325, - "packages/cli/tests/protected-input-forms.test.ts": 2810, - "packages/cli/tests/repl-admission.test.ts": 625, - "packages/cli/tests/repl-agent-execution.test.ts": 6220, - "packages/cli/tests/repl-agent-interface.test.ts": 38482, - "packages/cli/tests/repl-agent-journey.test.ts": 26815, - "packages/cli/tests/repl-asset-import.test.ts": 10711, - "packages/cli/tests/repl-boundaries.test.ts": 3465, - "packages/cli/tests/repl-caret.test.ts": 2254, - "packages/cli/tests/repl-composition.test.ts": 4756, - "packages/cli/tests/repl-entries.test.ts": 10380, - "packages/cli/tests/repl-exec-output.test.ts": 7721, - "packages/cli/tests/repl-execution.test.ts": 3973, - "packages/cli/tests/repl-exit-pty.test.ts": 8734, - "packages/cli/tests/repl-field-ownership.test.ts": 2226, - "packages/cli/tests/repl-focus-navigation.test.ts": 612, - "packages/cli/tests/repl-focus-regions.test.ts": 12276, - "packages/cli/tests/repl-forms.test.ts": 7994, - "packages/cli/tests/repl-git-observations.test.ts": 4482, - "packages/cli/tests/repl-headless-attach.test.ts": 83511, - "packages/cli/tests/repl-headless-clients.test.ts": 105241, - "packages/cli/tests/repl-headless-submit-process.test.ts": 16078, - "packages/cli/tests/repl-headless-submit.test.ts": 36092, - "packages/cli/tests/repl-headless.test.ts": 21523, - "packages/cli/tests/repl-history.test.ts": 4688, - "packages/cli/tests/repl-include-pty.test.ts": 43095, - "packages/cli/tests/repl-issues-pty.test.ts": 5682, - "packages/cli/tests/repl-journey.test.ts": 60623, - "packages/cli/tests/repl-layout-admission.test.ts": 11484, - "packages/cli/tests/repl-lifecycle.test.ts": 2174, - "packages/cli/tests/repl-model.test.ts": 3791, - "packages/cli/tests/repl-mounts-pty.test.ts": 45629, - "packages/cli/tests/repl-page.test.ts": 6305, - "packages/cli/tests/repl-plugin.test.ts": 5206, - "packages/cli/tests/repl-presentation-style.test.ts": 53421, - "packages/cli/tests/repl-project-files.test.ts": 18164, - "packages/cli/tests/repl-project-recovery.test.ts": 10196, - "packages/cli/tests/repl-push-pty.test.ts": 11447, - "packages/cli/tests/repl-repositories-pty.test.ts": 15607, - "packages/cli/tests/repl-retained-history.test.ts": 3016, - "packages/cli/tests/repl-route.test.ts": 3104, - "packages/cli/tests/repl-sidekick-acp.test.ts": 883536, - "packages/cli/tests/repl-sidekick-configuration.test.ts": 29648, - "packages/cli/tests/repl-sidekick-controls.test.ts": 67661, - "packages/cli/tests/repl-sidekick-dispatch.test.ts": 13359, - "packages/cli/tests/repl-sidekick-journey.test.ts": 41723, - "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 21916, - "packages/cli/tests/repl-sidekick-output.test.ts": 574, - "packages/cli/tests/repl-sidekick-presentation.test.ts": 88962, - "packages/cli/tests/repl-sidekick-recovery.test.ts": 235052, - "packages/cli/tests/repl-sidekick-storage.test.ts": 1584, - "packages/cli/tests/repl-sidekick-testing.test.ts": 27543, - "packages/cli/tests/repl-sidekick-typing.test.ts": 25139, - "packages/cli/tests/repl-source-reading.test.ts": 5610, - "packages/cli/tests/repl-terminal.test.ts": 17766, - "packages/cli/tests/repl-unfinished-entry.test.ts": 25379, - "packages/cli/tests/repl-wheel-frame.test.ts": 3242, - "packages/cli/tests/repl-wheel-input.test.ts": 2366, - "packages/cli/tests/repl-wheel-migration.test.ts": 45729, - "packages/cli/tests/repl-wheel-scroll.test.ts": 38440, - "packages/cli/tests/retained-mount-files.test.ts": 2951, - "packages/cli/tests/retained-mount-records.test.ts": 2343, - "packages/cli/tests/retained-repository-observations.test.ts": 7931, - "packages/cli/tests/retained-repository-pull-request-records.test.ts": 1611, - "packages/cli/tests/retained-repository-push-records.test.ts": 1533, - "packages/cli/tests/retained-repository-records.test.ts": 1495, - "packages/cli/tests/run-composition.test.ts": 9271, - "packages/cli/tests/run-deadline.test.ts": 4319, - "packages/cli/tests/run-profile.test.ts": 1834, - "packages/cli/tests/run-timeouts.test.ts": 30433, - "packages/cli/tests/secret-detection-cli.test.ts": 59996, - "packages/cli/tests/service-document.test.ts": 2859, - "packages/cli/tests/service-host.test.ts": 2495, - "packages/cli/tests/session-launch-cli.test.ts": 17293, - "packages/cli/tests/session-ownership-cli.test.ts": 11137, - "packages/cli/tests/stdin-cli.test.ts": 99960, - "packages/cli/tests/stdout-delivery.test.ts": 534, - "packages/cli/tests/syntax-cli.test.ts": 96130, - "packages/cli/tests/targets-cli.test.ts": 83464, - "packages/cli/tests/test-root-profile.test.ts": 11028, - "packages/cli/tests/test-target.test.ts": 116676, - "packages/cli/tests/testing-activation.test.ts": 1645, - "packages/cli/tests/testing-execution-host.test.ts": 148364, - "packages/cli/tests/upgrade-cli.test.ts": 77757, - "packages/cli/tests/upgrade-command-document.test.ts": 6381, - "packages/cli/tests/upgrade-output.test.ts": 2099, - "packages/cli/tests/value-root.test.ts": 48999, - "packages/cli/tests/verbose-component.test.ts": 2329, - "packages/cli/tests/workflow-activation.test.ts": 136594, - "packages/cli/tests/workflow-declaration.test.ts": 1588, - "packages/cli/tests/workflow-host.test.ts": 18187, - "packages/cli/tests/workspace-mounts.test.ts": 13847, - "packages/code-review-agent/tests/doctor.test.ts": 520, - "packages/code-review-agent/tests/parse-diagnostics.test.ts": 552, - "packages/code-review-agent/tests/parse-diff.test.ts": 539, - "packages/code-review-agent/tests/parse-doctor.test.ts": 1312, - "packages/code-review-agent/tests/parse-oxlint.test.ts": 526, - "packages/code-review-agent/tests/policy.test.ts": 522, - "packages/core/tests/agent-api.test.ts": 534, - "packages/core/tests/agent-components.test.ts": 2207, - "packages/core/tests/agent-conversation.test.ts": 4721, - "packages/core/tests/agent-function-components.test.ts": 5270, - "packages/core/tests/agent-session-launch.test.ts": 4857, - "packages/core/tests/agent-session-setup.test.ts": 541, - "packages/core/tests/agent-session-use.test.ts": 522, - "packages/core/tests/agent-stack.test.ts": 1644, - "packages/core/tests/all.test.ts": 4060, - "packages/core/tests/answer-identity.test.ts": 583, - "packages/core/tests/answers-component.test.ts": 2766, - "packages/core/tests/answers-expansion-recursion.test.ts": 1049, - "packages/core/tests/capture-errors.test.ts": 1323, - "packages/core/tests/capture-props.test.ts": 1070, - "packages/core/tests/cli-journal.test.ts": 34878, - "packages/core/tests/code-block-component.test.ts": 3483, - "packages/core/tests/compiler-boundary.test.ts": 1281, - "packages/core/tests/component-api.test.ts": 594, - "packages/core/tests/component-expansion.test.ts": 1382, - "packages/core/tests/component-invocation-hooks.test.ts": 1972, - "packages/core/tests/component-module-loader.test.ts": 1955, - "packages/core/tests/component-registration.test.ts": 2616, - "packages/core/tests/component-returns.test.ts": 3590, - "packages/core/tests/component-schema-conformance.test.ts": 1142, - "packages/core/tests/config-api.test.ts": 1424, - "packages/core/tests/construct-error-observation.test.ts": 1498, - "packages/core/tests/contextual-cwd.test.ts": 3080, - "packages/core/tests/daemon-integration.test.ts": 2084, - "packages/core/tests/daemon.test.ts": 621, - "packages/core/tests/declared-markdown-component.test.ts": 3839, - "packages/core/tests/document-output-api.test.ts": 538, - "packages/core/tests/document-target-execution.test.ts": 5420, - "packages/core/tests/document-targets.test.ts": 1265, - "packages/core/tests/document-validation.test.ts": 3278, - "packages/core/tests/documentation-index.test.ts": 944, - "packages/core/tests/each.test.ts": 1459, - "packages/core/tests/elicit-component.test.ts": 2304, - "packages/core/tests/ephemeral-service.test.ts": 2068, - "packages/core/tests/eval-bindings.test.ts": 1565, - "packages/core/tests/eval-context.test.ts": 652, - "packages/core/tests/eval-durable.test.ts": 1522, - "packages/core/tests/eval-error-mode.test.ts": 1845, - "packages/core/tests/eval-interpolate.test.ts": 559, - "packages/core/tests/eval-middleware.test.ts": 683, - "packages/core/tests/eval-persist.test.ts": 1525, - "packages/core/tests/eval-return.test.ts": 1417, - "packages/core/tests/eval-scope.test.ts": 1294, - "packages/core/tests/eval-timeout.test.ts": 1385, - "packages/core/tests/eval-transform.test.ts": 623, - "packages/core/tests/evaluate-component.test.ts": 4590, - "packages/core/tests/evaluate-loaded-copy.test.ts": 1488, - "packages/core/tests/evaluate-provider-lifetime.test.ts": 1574, - "packages/core/tests/evaluation-profile.test.ts": 920, - "packages/core/tests/exec-timeout.test.ts": 1486, - "packages/core/tests/execute.test.ts": 2441, - "packages/core/tests/execution-init.test.ts": 1578, - "packages/core/tests/execution-protocol.test.ts": 3331, - "packages/core/tests/expand.test.ts": 3070, - "packages/core/tests/expansion-identity.test.ts": 2123, - "packages/core/tests/expansion-metadata.test.ts": 1089, - "packages/core/tests/expression-props.test.ts": 2090, - "packages/core/tests/fail-component.test.ts": 1778, - "packages/core/tests/failure-printing.test.ts": 1756, - "packages/core/tests/fatal-cause.test.ts": 951, - "packages/core/tests/fetch-component.test.ts": 3344, - "packages/core/tests/file-component.test.ts": 3741, - "packages/core/tests/file-delete-component.test.ts": 1692, - "packages/core/tests/files-fatal.test.ts": 2862, - "packages/core/tests/filesystem-placement.test.ts": 1875, - "packages/core/tests/fixed-filesystem-components.test.ts": 1947, - "packages/core/tests/foreground-exec.test.ts": 2744, - "packages/core/tests/frontmatter.test.ts": 603, - "packages/core/tests/function-components.test.ts": 2663, - "packages/core/tests/generated-composition.test.ts": 2348, - "packages/core/tests/generated-owner-metadata.test.ts": 1535, - "packages/core/tests/generated-selection.test.ts": 2161, - "packages/core/tests/generated-xmd-preview.test.ts": 1239, - "packages/core/tests/generated-xmd-root.test.ts": 3015, - "packages/core/tests/generated-xmd.test.ts": 3726, - "packages/core/tests/glob-api.test.ts": 635, - "packages/core/tests/glob-component.test.ts": 2389, - "packages/core/tests/guarded-journal.test.ts": 1540, - "packages/core/tests/has-content.test.ts": 1034, - "packages/core/tests/heal.test.ts": 1050, - "packages/core/tests/host-generated-root.test.ts": 2735, - "packages/core/tests/if.test.ts": 2139, - "packages/core/tests/inline-root.test.ts": 1533, - "packages/core/tests/invocation-failures.test.ts": 643, - "packages/core/tests/invocation-identity.test.ts": 2794, - "packages/core/tests/invocation-scope.test.ts": 1279, - "packages/core/tests/journal-source-position.test.ts": 1575, - "packages/core/tests/json-component.test.ts": 1672, - "packages/core/tests/json.test.ts": 537, - "packages/core/tests/let.test.ts": 1902, - "packages/core/tests/loop.test.ts": 2726, - "packages/core/tests/named-slots.test.ts": 2545, - "packages/core/tests/output-error-mode.test.ts": 3114, - "packages/core/tests/output-normalize.test.ts": 575, - "packages/core/tests/output-terminal.test.ts": 782, - "packages/core/tests/parse-components.test.ts": 1660, - "packages/core/tests/plan-response.test.ts": 643, - "packages/core/tests/plugin-api.test.ts": 1148, - "packages/core/tests/plugin.test.ts": 1119, - "packages/core/tests/props-binding.test.ts": 1649, - "packages/core/tests/protected-content.test.ts": 616, - "packages/core/tests/protected-input-replay.test.ts": 2126, - "packages/core/tests/protected-input.test.ts": 2744, - "packages/core/tests/query-component.test.ts": 1713, - "packages/core/tests/registered-printed-errors.test.ts": 959, - "packages/core/tests/replay-stream.test.ts": 551, - "packages/core/tests/retain.test.ts": 1194, - "packages/core/tests/root-composition.test.ts": 1602, - "packages/core/tests/root-props.test.ts": 1657, - "packages/core/tests/root-provider.test.ts": 1505, - "packages/core/tests/sample-component.test.ts": 2679, - "packages/core/tests/scanner.test.ts": 641, - "packages/core/tests/scope-local.test.ts": 540, - "packages/core/tests/secret-detection.test.ts": 2376, - "packages/core/tests/secret-files.test.ts": 794, - "packages/core/tests/secret-rules.test.ts": 890, - "packages/core/tests/secret-scanner-baseline.test.ts": 784, - "packages/core/tests/secret-scanner.test.ts": 729, - "packages/core/tests/shared-operation-permissions.test.ts": 2700, - "packages/core/tests/slice-component.test.ts": 2474, - "packages/core/tests/source-inspection.test.ts": 637, - "packages/core/tests/source-position.test.ts": 1324, - "packages/core/tests/state-ownership.test.ts": 1362, - "packages/core/tests/streaming-emission.test.ts": 1324, - "packages/core/tests/switch.test.ts": 2120, - "packages/core/tests/syntax-api.test.ts": 1748, - "packages/core/tests/syntax-catalog.test.ts": 2029, - "packages/core/tests/syntax-component.test.ts": 3268, - "packages/core/tests/temp-dir.test.ts": 4121, - "packages/core/tests/temp-file-compiler.test.ts": 597, - "packages/core/tests/text-interpolation.test.ts": 1020, - "packages/core/tests/try-content.test.ts": 1017, - "packages/core/tests/typescript-materialization.test.ts": 4025, - "packages/core/tests/unused-in-diff.test.ts": 1563, - "packages/core/tests/validate.test.ts": 904, - "packages/core/tests/validation-integration.test.ts": 1386, - "packages/core/tests/workflow-component-bundle.test.ts": 1839, - "packages/core/tests/yaml-component.test.ts": 1173, - "packages/durable-streams/tests/context.test.ts": 589, - "packages/durable-streams/tests/deterministic-id.test.ts": 632, - "packages/durable-streams/tests/divergence-api.test.ts": 604, - "packages/durable-streams/tests/divergence.test.ts": 639, - "packages/durable-streams/tests/durable-each.test.ts": 662, - "packages/durable-streams/tests/durable-position.test.ts": 633, - "packages/durable-streams/tests/durable-run.test.ts": 679, - "packages/durable-streams/tests/ephemeral.test.ts": 627, - "packages/durable-streams/tests/fail-stop.test.ts": 682, - "packages/durable-streams/tests/guard-stream.test.ts": 661, - "packages/durable-streams/tests/live-coordinator.test.ts": 617, - "packages/durable-streams/tests/parse.test.ts": 645, - "packages/durable-streams/tests/replay-guard.test.ts": 655, - "packages/durable-streams/tests/replay-index.test.ts": 610, - "packages/durable-streams/tests/retained.test.ts": 592, - "packages/durable-streams/tests/serialize-event.test.ts": 594, - "packages/durable-streams/tests/smoke.test.ts": 598, - "packages/durable-streams/tests/structured-concurrency.test.ts": 684, - "packages/durable-streams/tests/terminal-boundary.test.ts": 656, - "packages/durable-streams/tests/types.test.ts": 594, - "packages/git/tests/api-entrypoint.test.ts": 1491, - "packages/git/tests/composition-component-binding.test.ts": 2514, - "packages/git/tests/git-commit-read.test.ts": 12340, - "packages/git/tests/git-host-effect.test.ts": 9486, - "packages/git/tests/git-host-owned.test.ts": 2748, - "packages/git/tests/git-observation-records.test.ts": 1444, - "packages/git/tests/git-observations.test.ts": 24020, - "packages/git/tests/git.test.ts": 1459, - "packages/git/tests/github-activation.test.ts": 1911, - "packages/git/tests/github-issues.test.ts": 1451, - "packages/git/tests/github-pull-requests.test.ts": 1438, - "packages/git/tests/issue-markdown.test.ts": 11490, - "packages/git/tests/issue-owned.test.ts": 4256, - "packages/git/tests/issue-records.test.ts": 1380, - "packages/git/tests/module-partition.test.ts": 3603, - "packages/git/tests/plugin.test.ts": 2048, - "packages/git/tests/provider-neutrality.test.ts": 2590, - "packages/git/tests/pull-request-records.test.ts": 1393, - "packages/git/tests/run-composition-lazy.test.ts": 2096, - "packages/git/tests/selection-authentication.test.ts": 561, - "packages/runtime/tests/duration.test.ts": 626, - "packages/runtime/tests/fetch.test.ts": 1220, - "packages/runtime/tests/files-call.test.ts": 556, - "packages/runtime/tests/fs.test.ts": 601, - "packages/runtime/tests/host-files.test.ts": 1394, - "packages/runtime/tests/native-launcher.test.ts": 2949, - "packages/runtime/tests/permissions.test.ts": 582, - "packages/runtime/tests/process.test.ts": 604, - "packages/runtime/tests/service.test.ts": 585, - "packages/test-agent/tests/acp-server.test.ts": 726, - "packages/test-agent/tests/behavior-engine.test.ts": 1769, - "packages/test-agent/tests/bridge.test.ts": 555, - "packages/test-agent/tests/command-lazy.test.ts": 9827, - "packages/test-agent/tests/components.test.ts": 186530, - "packages/test-agent/tests/controller.test.ts": 871, - "packages/test-agent/tests/cross-package-resolution.test.ts": 3440, - "packages/test-agent/tests/native-launch.test.ts": 102057, - "packages/test-agent/tests/net.test.ts": 657, - "packages/test-agent/tests/profile.test.ts": 1343, - "packages/test-agent/tests/protocol.test.ts": 634, - "packages/test-agent/tests/provider.test.ts": 1719, - "packages/test-agent/tests/public-api.test.ts": 1903, - "packages/test-agent/tests/route-slot.test.ts": 543, - "packages/test-agent/tests/session-configuration.test.ts": 268081, - "packages/test-agent/tests/smoke.test.ts": 45736, - "packages/test-agent/tests/template.test.ts": 1390, - "packages/test-agent/tests/worker-lifecycle.test.ts": 51732, - "packages/test-support/tests/journal.test.ts": 578, - "packages/testing/tests/agent-composition.test.ts": 1682, - "packages/testing/tests/assert-throws.test.ts": 3460, - "packages/testing/tests/assert.test.ts": 576, - "packages/testing/tests/assertions.test.ts": 4329, - "packages/testing/tests/boundary-contract.test.ts": 2809, - "packages/testing/tests/cli.test.ts": 25077, - "packages/testing/tests/execution-harness.test.ts": 8528, - "packages/testing/tests/replay.test.ts": 3451, - "packages/testing/tests/smoke.test.ts": 4451, - "packages/testing/tests/test-component.test.ts": 4453, - "packages/testing/tests/testing-activation.test.ts": 3863, - "packages/testing/tests/testing-boundary.test.ts": 2304, - "packages/testing/tests/testing-mode.test.ts": 4544, - "packages/testing/tests/use-testing.test.ts": 3044, - "packages/web/tests/assets.test.ts": 573, - "packages/web/tests/client-logic.test.ts": 604, - "packages/web/tests/compile.test.ts": 2183, - "packages/web/tests/component.test.ts": 3172, - "packages/web/tests/declaration.test.ts": 1868, - "packages/web/tests/document.test.ts": 2024, - "packages/web/tests/elicitation.test.ts": 1838, - "packages/web/tests/live-form.test.ts": 1699, - "packages/web/tests/markdown.test.ts": 762, - "packages/web/tests/opener.test.ts": 1677, - "packages/web/tests/page.test.ts": 567, - "packages/web/tests/responder.test.ts": 2149, - "packages/web/tests/server-lifecycle.test.ts": 1920, - "packages/web/tests/server.test.ts": 2255, - "packages/workflow/tests/generated-observations.test.ts": 2180, - "packages/workflow/tests/retained-run.test.ts": 2604, - "packages/workflow/tests/service-denial.test.ts": 615, - "packages/workflow/tests/workflow-bundle.test.ts": 1927, - "packages/workflow/tests/workflow-definition.test.ts": 1487, - "packages/workflow/tests/workflow-fork.test.ts": 1404, - "packages/workflow/tests/workflow-run.test.ts": 2266, - "packages/workflow/tests/workspace-effect.test.ts": 3469, - "packages/workflow/tests/xmd-artifact.test.ts": 4206, - "scripts/tests/adapter-vendor.test.ts": 1238, - "scripts/tests/bootstrap-npm-package.test.ts": 11255, - "scripts/tests/bump-version.test.ts": 692, - "scripts/tests/changed-paths.test.ts": 588, - "scripts/tests/ci-workflow.test.ts": 4797, - "scripts/tests/claude-preparation.test.ts": 1511, - "scripts/tests/claude-snapshot-module.test.ts": 946, - "scripts/tests/consumer-cycle.test.ts": 779, - "scripts/tests/filesystem-contract-workflow.test.ts": 664, - "scripts/tests/main-green.test.ts": 826, - "scripts/tests/main-health.test.ts": 686, - "scripts/tests/no-module-scoped-registry.test.ts": 1596, - "scripts/tests/no-redundant-test-scope.test.ts": 6437, - "scripts/tests/no-sync-filesystem.test.ts": 6998, - "scripts/tests/no-yield-in-finally.test.ts": 1551, - "scripts/tests/oxlint-policy.test.ts": 16343, - "scripts/tests/packaged-document.test.ts": 703, - "scripts/tests/prefer-effection-operation.test.ts": 12791, - "scripts/tests/prefer-effection-result.test.ts": 5473, - "scripts/tests/publish-workflow-membership.test.ts": 729, - "scripts/tests/review-infrastructure.test.ts": 4628, - "scripts/tests/runtime-exclusions.test.ts": 1231, - "scripts/tests/runtime-tests.test.ts": 629, - "scripts/tests/scope-bound-event-registration.test.ts": 6595, - "scripts/tests/staged-write.test.ts": 613, - "scripts/tests/test-file-discovery.test.ts": 782, - "scripts/tests/test-shards.test.ts": 612, - "scripts/tests/test-weights.test.ts": 951, - "scripts/tests/tracked.test.ts": 596, - "scripts/tests/verify-coordinator.test.ts": 806, - "scripts/tests/version-lockstep.test.ts": 802, - "scripts/tests/web-client-module.test.ts": 698, - "scripts/tests/workspace.test.ts": 613 + "packages/acp/tests/acceptance.test.ts": 1561, + "packages/acp/tests/acpx-checkpoint-meta.test.ts": 5565, + "packages/acp/tests/acpx-materialization.test.ts": 17848, + "packages/acp/tests/acpx-session-save.test.ts": 753, + "packages/acp/tests/acpx-transient-env.test.ts": 1351, + "packages/acp/tests/adapter-materialization.test.ts": 63476, + "packages/acp/tests/adapter-protocol.test.ts": 33632, + "packages/acp/tests/agent-certification-report.test.ts": 345, + "packages/acp/tests/agent-certification.test.ts": 24330, + "packages/acp/tests/codex-surfaces.test.ts": 392, + "packages/acp/tests/host-conversation.test.ts": 6184, + "packages/acp/tests/native-launch.test.ts": 1499, + "packages/acp/tests/permission-bridge.test.ts": 795, + "packages/acp/tests/prompt-usage.test.ts": 8820, + "packages/acp/tests/provider.test.ts": 1604, + "packages/acp/tests/released-runtime.test.ts": 27216, + "packages/acp/tests/serial-queue.test.ts": 320, + "packages/acp/tests/session-key.test.ts": 527, + "packages/acp/tests/terminal-screen.test.ts": 367, + "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 12471, + "packages/cli/tests/adversarial-artifact-return.test.ts": 25935, + "packages/cli/tests/adversarial-delivery.test.ts": 28054, + "packages/cli/tests/adversarial-friction.test.ts": 73050, + "packages/cli/tests/adversarial-implementation.test.ts": 70142, + "packages/cli/tests/adversarial-launch-policy.test.ts": 52418, + "packages/cli/tests/adversarial-preparation.test.ts": 120913, + "packages/cli/tests/adversarial-production-role.test.ts": 65156, + "packages/cli/tests/adversarial-session-continuity.test.ts": 9880, + "packages/cli/tests/adversarial-workflow.test.ts": 56913, + "packages/cli/tests/agent-adapters.test.ts": 3252, + "packages/cli/tests/agent-certification.test.ts": 200944, + "packages/cli/tests/agent-cli.test.ts": 35941, + "packages/cli/tests/agent-config.test.ts": 334, + "packages/cli/tests/agent-options-cli.test.ts": 16664, + "packages/cli/tests/agent-options.test.ts": 332, + "packages/cli/tests/agent-session-coordinator.test.ts": 1063, + "packages/cli/tests/cli-help.test.ts": 105333, + "packages/cli/tests/command.test.ts": 16480, + "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 3133, + "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 156663, + "packages/cli/tests/document-suites/props/props-markdown.test.ts": 2300, + "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 7224, + "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 3820, + "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 14035, + "packages/cli/tests/evaluate-component.test.ts": 11668, + "packages/cli/tests/fetch-cli.test.ts": 3756, + "packages/cli/tests/file-stream.test.ts": 334, + "packages/cli/tests/inline-cli.test.ts": 53599, + "packages/cli/tests/launch-timeout.test.ts": 20347, + "packages/cli/tests/packaged-document.test.ts": 1115, + "packages/cli/tests/parse-failure.test.ts": 5572, + "packages/cli/tests/plan-args.test.ts": 322, + "packages/cli/tests/plan-cli.test.ts": 95619, + "packages/cli/tests/plan-command-document.test.ts": 5400, + "packages/cli/tests/plan-component.test.ts": 7038, + "packages/cli/tests/plan-host-acts.test.ts": 1455, + "packages/cli/tests/plan.test.ts": 9490, + "packages/cli/tests/plugin-cli.test.ts": 64755, + "packages/cli/tests/plugin-host.test.ts": 2016, + "packages/cli/tests/plugin-modules.test.ts": 1761, + "packages/cli/tests/plugin-selection.test.ts": 318, + "packages/cli/tests/process-retention.test.ts": 3768, + "packages/cli/tests/props-cli.test.ts": 50954, + "packages/cli/tests/props-schema.test.ts": 774, + "packages/cli/tests/props-sources.test.ts": 420, + "packages/cli/tests/protected-input-cli.test.ts": 11565, + "packages/cli/tests/protected-input-forms.test.ts": 1500, + "packages/cli/tests/repl-admission.test.ts": 344, + "packages/cli/tests/repl-agent-execution.test.ts": 3152, + "packages/cli/tests/repl-agent-interface.test.ts": 30399, + "packages/cli/tests/repl-agent-journey.test.ts": 14558, + "packages/cli/tests/repl-asset-import.test.ts": 5819, + "packages/cli/tests/repl-boundaries.test.ts": 1801, + "packages/cli/tests/repl-caret.test.ts": 1220, + "packages/cli/tests/repl-composition.test.ts": 2637, + "packages/cli/tests/repl-entries.test.ts": 8090, + "packages/cli/tests/repl-exec-output.test.ts": 4402, + "packages/cli/tests/repl-execution.test.ts": 2231, + "packages/cli/tests/repl-exit-pty.test.ts": 4973, + "packages/cli/tests/repl-field-ownership.test.ts": 1288, + "packages/cli/tests/repl-focus-navigation.test.ts": 374, + "packages/cli/tests/repl-focus-regions.test.ts": 8886, + "packages/cli/tests/repl-forms.test.ts": 4772, + "packages/cli/tests/repl-git-observations.test.ts": 2623, + "packages/cli/tests/repl-headless-attach.test.ts": 48792, + "packages/cli/tests/repl-headless-clients.test.ts": 58880, + "packages/cli/tests/repl-headless-submit-process.test.ts": 8171, + "packages/cli/tests/repl-headless-submit.test.ts": 19037, + "packages/cli/tests/repl-headless.test.ts": 11367, + "packages/cli/tests/repl-history.test.ts": 2290, + "packages/cli/tests/repl-include-pty.test.ts": 28254, + "packages/cli/tests/repl-issues-pty.test.ts": 3364, + "packages/cli/tests/repl-journey.test.ts": 33670, + "packages/cli/tests/repl-layout-admission.test.ts": 6848, + "packages/cli/tests/repl-lifecycle.test.ts": 1174, + "packages/cli/tests/repl-model.test.ts": 2052, + "packages/cli/tests/repl-mounts-pty.test.ts": 28115, + "packages/cli/tests/repl-page.test.ts": 3520, + "packages/cli/tests/repl-parse-failure.test.ts": 941, + "packages/cli/tests/repl-plugin.test.ts": 2730, + "packages/cli/tests/repl-presentation-style.test.ts": 27916, + "packages/cli/tests/repl-project-files.test.ts": 10151, + "packages/cli/tests/repl-project-recovery.test.ts": 5880, + "packages/cli/tests/repl-push-pty.test.ts": 6827, + "packages/cli/tests/repl-repositories-pty.test.ts": 9637, + "packages/cli/tests/repl-retained-history.test.ts": 1732, + "packages/cli/tests/repl-route.test.ts": 1681, + "packages/cli/tests/repl-sidekick-acp.test.ts": 456095, + "packages/cli/tests/repl-sidekick-configuration.test.ts": 16412, + "packages/cli/tests/repl-sidekick-controls.test.ts": 34638, + "packages/cli/tests/repl-sidekick-dispatch.test.ts": 7936, + "packages/cli/tests/repl-sidekick-journey.test.ts": 23794, + "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 13460, + "packages/cli/tests/repl-sidekick-output.test.ts": 340, + "packages/cli/tests/repl-sidekick-presentation.test.ts": 73122, + "packages/cli/tests/repl-sidekick-recovery.test.ts": 124329, + "packages/cli/tests/repl-sidekick-storage.test.ts": 1073, + "packages/cli/tests/repl-sidekick-testing.test.ts": 15572, + "packages/cli/tests/repl-sidekick-typing.test.ts": 13809, + "packages/cli/tests/repl-source-reading.test.ts": 2837, + "packages/cli/tests/repl-task-tabs.test.ts": 4986, + "packages/cli/tests/repl-terminal.test.ts": 10306, + "packages/cli/tests/repl-todo-context.test.ts": 2452, + "packages/cli/tests/repl-todo-journey.test.ts": 9014, + "packages/cli/tests/repl-todo-recovery.test.ts": 2586, + "packages/cli/tests/repl-todo.test.ts": 1009, + "packages/cli/tests/repl-unfinished-entry.test.ts": 14058, + "packages/cli/tests/repl-wheel-frame.test.ts": 1762, + "packages/cli/tests/repl-wheel-input.test.ts": 1263, + "packages/cli/tests/repl-wheel-migration.test.ts": 30776, + "packages/cli/tests/repl-wheel-scroll.test.ts": 26900, + "packages/cli/tests/retained-mount-files.test.ts": 1598, + "packages/cli/tests/retained-mount-records.test.ts": 1243, + "packages/cli/tests/retained-repository-observations.test.ts": 4800, + "packages/cli/tests/retained-repository-pull-request-records.test.ts": 853, + "packages/cli/tests/retained-repository-push-records.test.ts": 833, + "packages/cli/tests/retained-repository-records.test.ts": 819, + "packages/cli/tests/run-composition.test.ts": 4757, + "packages/cli/tests/run-deadline.test.ts": 2870, + "packages/cli/tests/run-profile.test.ts": 990, + "packages/cli/tests/run-timeouts.test.ts": 17085, + "packages/cli/tests/secret-detection-cli.test.ts": 29802, + "packages/cli/tests/service-document.test.ts": 1646, + "packages/cli/tests/service-host.test.ts": 1495, + "packages/cli/tests/session-launch-cli.test.ts": 8679, + "packages/cli/tests/session-ownership-cli.test.ts": 5558, + "packages/cli/tests/stdin-cli.test.ts": 50911, + "packages/cli/tests/stdout-delivery.test.ts": 318, + "packages/cli/tests/syntax-cli.test.ts": 47429, + "packages/cli/tests/targets-cli.test.ts": 41279, + "packages/cli/tests/test-root-profile.test.ts": 5436, + "packages/cli/tests/test-target.test.ts": 57475, + "packages/cli/tests/testing-activation.test.ts": 880, + "packages/cli/tests/testing-execution-host.test.ts": 77514, + "packages/cli/tests/upgrade-cli.test.ts": 38039, + "packages/cli/tests/upgrade-command-document.test.ts": 3440, + "packages/cli/tests/upgrade-output.test.ts": 1126, + "packages/cli/tests/value-root.test.ts": 24474, + "packages/cli/tests/verbose-component.test.ts": 1242, + "packages/cli/tests/workflow-activation.test.ts": 67231, + "packages/cli/tests/workflow-declaration.test.ts": 886, + "packages/cli/tests/workflow-host.test.ts": 8934, + "packages/cli/tests/workspace-mounts.test.ts": 6964, + "packages/code-review-agent/tests/doctor.test.ts": 315, + "packages/code-review-agent/tests/parse-diagnostics.test.ts": 325, + "packages/code-review-agent/tests/parse-diff.test.ts": 327, + "packages/code-review-agent/tests/parse-doctor.test.ts": 728, + "packages/code-review-agent/tests/parse-oxlint.test.ts": 318, + "packages/code-review-agent/tests/policy.test.ts": 314, + "packages/core/tests/agent-api.test.ts": 319, + "packages/core/tests/agent-components.test.ts": 1251, + "packages/core/tests/agent-conversation.test.ts": 2905, + "packages/core/tests/agent-function-components.test.ts": 2810, + "packages/core/tests/agent-session-launch.test.ts": 2438, + "packages/core/tests/agent-session-setup.test.ts": 322, + "packages/core/tests/agent-session-use.test.ts": 306, + "packages/core/tests/agent-stack.test.ts": 917, + "packages/core/tests/agent-usage.test.ts": 1026, + "packages/core/tests/all.test.ts": 2136, + "packages/core/tests/answer-identity.test.ts": 347, + "packages/core/tests/answers-component.test.ts": 1472, + "packages/core/tests/answers-expansion-recursion.test.ts": 584, + "packages/core/tests/capture-errors.test.ts": 730, + "packages/core/tests/capture-props.test.ts": 579, + "packages/core/tests/cli-journal.test.ts": 17332, + "packages/core/tests/code-block-component.test.ts": 1820, + "packages/core/tests/compiler-boundary.test.ts": 701, + "packages/core/tests/component-api.test.ts": 356, + "packages/core/tests/component-expansion.test.ts": 746, + "packages/core/tests/component-invocation-hooks.test.ts": 1082, + "packages/core/tests/component-module-loader.test.ts": 1101, + "packages/core/tests/component-registration.test.ts": 1441, + "packages/core/tests/component-returns.test.ts": 2009, + "packages/core/tests/component-schema-conformance.test.ts": 638, + "packages/core/tests/config-api.test.ts": 776, + "packages/core/tests/construct-error-observation.test.ts": 776, + "packages/core/tests/contextual-cwd.test.ts": 2154, + "packages/core/tests/daemon-integration.test.ts": 1379, + "packages/core/tests/daemon.test.ts": 367, + "packages/core/tests/declared-markdown-component.test.ts": 1888, + "packages/core/tests/document-output-api.test.ts": 330, + "packages/core/tests/document-target-execution.test.ts": 2775, + "packages/core/tests/document-targets.test.ts": 697, + "packages/core/tests/document-validation.test.ts": 1694, + "packages/core/tests/documentation-index.test.ts": 521, + "packages/core/tests/each.test.ts": 774, + "packages/core/tests/elicit-component.test.ts": 1242, + "packages/core/tests/ephemeral-service.test.ts": 1117, + "packages/core/tests/eval-bindings.test.ts": 863, + "packages/core/tests/eval-context.test.ts": 373, + "packages/core/tests/eval-durable.test.ts": 827, + "packages/core/tests/eval-error-mode.test.ts": 1026, + "packages/core/tests/eval-interpolate.test.ts": 331, + "packages/core/tests/eval-middleware.test.ts": 408, + "packages/core/tests/eval-persist.test.ts": 854, + "packages/core/tests/eval-return.test.ts": 793, + "packages/core/tests/eval-scope.test.ts": 705, + "packages/core/tests/eval-timeout.test.ts": 791, + "packages/core/tests/eval-transform.test.ts": 355, + "packages/core/tests/evaluate-component.test.ts": 2439, + "packages/core/tests/evaluate-loaded-copy.test.ts": 840, + "packages/core/tests/evaluate-provider-lifetime.test.ts": 864, + "packages/core/tests/evaluation-profile.test.ts": 524, + "packages/core/tests/exec-timeout.test.ts": 815, + "packages/core/tests/execute.test.ts": 1356, + "packages/core/tests/execution-init.test.ts": 887, + "packages/core/tests/execution-protocol.test.ts": 1816, + "packages/core/tests/expand.test.ts": 1640, + "packages/core/tests/expansion-identity.test.ts": 1184, + "packages/core/tests/expansion-metadata.test.ts": 597, + "packages/core/tests/expression-props.test.ts": 1170, + "packages/core/tests/fail-component.test.ts": 987, + "packages/core/tests/failure-printing.test.ts": 897, + "packages/core/tests/fatal-cause.test.ts": 530, + "packages/core/tests/fetch-component.test.ts": 1873, + "packages/core/tests/file-component.test.ts": 2427, + "packages/core/tests/file-delete-component.test.ts": 888, + "packages/core/tests/files-fatal.test.ts": 1500, + "packages/core/tests/filesystem-placement.test.ts": 1021, + "packages/core/tests/fixed-filesystem-components.test.ts": 1024, + "packages/core/tests/foreground-exec.test.ts": 1508, + "packages/core/tests/frontmatter.test.ts": 351, + "packages/core/tests/function-components.test.ts": 1477, + "packages/core/tests/generated-composition.test.ts": 1268, + "packages/core/tests/generated-owner-metadata.test.ts": 819, + "packages/core/tests/generated-selection.test.ts": 1186, + "packages/core/tests/generated-xmd-preview.test.ts": 671, + "packages/core/tests/generated-xmd-root.test.ts": 1572, + "packages/core/tests/generated-xmd.test.ts": 2008, + "packages/core/tests/glob-api.test.ts": 367, + "packages/core/tests/glob-component.test.ts": 1364, + "packages/core/tests/guarded-journal.test.ts": 830, + "packages/core/tests/has-content.test.ts": 556, + "packages/core/tests/heal.test.ts": 561, + "packages/core/tests/host-generated-root.test.ts": 1418, + "packages/core/tests/if.test.ts": 1130, + "packages/core/tests/inline-root.test.ts": 818, + "packages/core/tests/invocation-failures.test.ts": 376, + "packages/core/tests/invocation-identity.test.ts": 1518, + "packages/core/tests/invocation-scope.test.ts": 727, + "packages/core/tests/journal-source-position.test.ts": 869, + "packages/core/tests/json-component.test.ts": 921, + "packages/core/tests/json.test.ts": 329, + "packages/core/tests/let.test.ts": 980, + "packages/core/tests/loop.test.ts": 1391, + "packages/core/tests/named-slots.test.ts": 1355, + "packages/core/tests/output-error-mode.test.ts": 1654, + "packages/core/tests/output-normalize.test.ts": 337, + "packages/core/tests/output-terminal.test.ts": 434, + "packages/core/tests/parse-components.test.ts": 1000, + "packages/core/tests/plan-response.test.ts": 363, + "packages/core/tests/plugin-api.test.ts": 625, + "packages/core/tests/plugin.test.ts": 604, + "packages/core/tests/props-binding.test.ts": 896, + "packages/core/tests/protected-content.test.ts": 352, + "packages/core/tests/protected-input-replay.test.ts": 1177, + "packages/core/tests/protected-input.test.ts": 1461, + "packages/core/tests/query-component.test.ts": 923, + "packages/core/tests/registered-printed-errors.test.ts": 528, + "packages/core/tests/replay-stream.test.ts": 323, + "packages/core/tests/retain.test.ts": 680, + "packages/core/tests/root-composition.test.ts": 855, + "packages/core/tests/root-props.test.ts": 886, + "packages/core/tests/root-provider.test.ts": 829, + "packages/core/tests/sample-component.test.ts": 1477, + "packages/core/tests/scanner.test.ts": 360, + "packages/core/tests/scope-local.test.ts": 321, + "packages/core/tests/secret-detection.test.ts": 1384, + "packages/core/tests/secret-files.test.ts": 458, + "packages/core/tests/secret-rules.test.ts": 488, + "packages/core/tests/secret-scanner-baseline.test.ts": 454, + "packages/core/tests/secret-scanner.test.ts": 422, + "packages/core/tests/shared-operation-permissions.test.ts": 1465, + "packages/core/tests/slice-component.test.ts": 1328, + "packages/core/tests/source-inspection.test.ts": 378, + "packages/core/tests/source-position.test.ts": 695, + "packages/core/tests/state-ownership.test.ts": 734, + "packages/core/tests/streaming-emission.test.ts": 713, + "packages/core/tests/switch.test.ts": 1154, + "packages/core/tests/syntax-api.test.ts": 922, + "packages/core/tests/syntax-catalog.test.ts": 1073, + "packages/core/tests/syntax-component.test.ts": 1703, + "packages/core/tests/temp-dir.test.ts": 3104, + "packages/core/tests/temp-file-compiler.test.ts": 361, + "packages/core/tests/text-interpolation.test.ts": 541, + "packages/core/tests/try-content.test.ts": 551, + "packages/core/tests/typescript-materialization.test.ts": 2163, + "packages/core/tests/unused-in-diff.test.ts": 862, + "packages/core/tests/validate.test.ts": 483, + "packages/core/tests/validation-integration.test.ts": 731, + "packages/core/tests/workflow-component-bundle.test.ts": 999, + "packages/core/tests/yaml-component.test.ts": 614, + "packages/durable-streams/tests/context.test.ts": 337, + "packages/durable-streams/tests/deterministic-id.test.ts": 356, + "packages/durable-streams/tests/divergence-api.test.ts": 342, + "packages/durable-streams/tests/divergence.test.ts": 355, + "packages/durable-streams/tests/durable-each.test.ts": 362, + "packages/durable-streams/tests/durable-position.test.ts": 347, + "packages/durable-streams/tests/durable-run.test.ts": 382, + "packages/durable-streams/tests/ephemeral.test.ts": 358, + "packages/durable-streams/tests/fail-stop.test.ts": 373, + "packages/durable-streams/tests/guard-stream.test.ts": 375, + "packages/durable-streams/tests/live-coordinator.test.ts": 344, + "packages/durable-streams/tests/parse.test.ts": 353, + "packages/durable-streams/tests/replay-guard.test.ts": 448, + "packages/durable-streams/tests/replay-index.test.ts": 340, + "packages/durable-streams/tests/retained.test.ts": 341, + "packages/durable-streams/tests/serialize-event.test.ts": 336, + "packages/durable-streams/tests/smoke.test.ts": 337, + "packages/durable-streams/tests/structured-concurrency.test.ts": 377, + "packages/durable-streams/tests/terminal-boundary.test.ts": 370, + "packages/durable-streams/tests/types.test.ts": 306, + "packages/git/tests/api-entrypoint.test.ts": 804, + "packages/git/tests/composition-component-binding.test.ts": 1283, + "packages/git/tests/git-commit-read.test.ts": 6791, + "packages/git/tests/git-host-effect.test.ts": 5341, + "packages/git/tests/git-host-owned.test.ts": 1552, + "packages/git/tests/git-observation-records.test.ts": 783, + "packages/git/tests/git-observations.test.ts": 13777, + "packages/git/tests/git.test.ts": 811, + "packages/git/tests/github-activation.test.ts": 1011, + "packages/git/tests/github-issues.test.ts": 809, + "packages/git/tests/github-pull-requests.test.ts": 807, + "packages/git/tests/issue-markdown.test.ts": 5869, + "packages/git/tests/issue-owned.test.ts": 2095, + "packages/git/tests/issue-records.test.ts": 752, + "packages/git/tests/module-partition.test.ts": 1877, + "packages/git/tests/plugin.test.ts": 1075, + "packages/git/tests/provider-neutrality.test.ts": 1420, + "packages/git/tests/pull-request-records.test.ts": 771, + "packages/git/tests/run-composition-lazy.test.ts": 1147, + "packages/git/tests/selection-authentication.test.ts": 329, + "packages/runtime/tests/duration.test.ts": 327, + "packages/runtime/tests/fetch.test.ts": 853, + "packages/runtime/tests/files-call.test.ts": 326, + "packages/runtime/tests/fs.test.ts": 365, + "packages/runtime/tests/host-files.test.ts": 1024, + "packages/runtime/tests/native-launcher.test.ts": 2627, + "packages/runtime/tests/permissions.test.ts": 339, + "packages/runtime/tests/process.test.ts": 358, + "packages/runtime/tests/service.test.ts": 347, + "packages/test-agent/tests/acp-server.test.ts": 427, + "packages/test-agent/tests/behavior-engine.test.ts": 973, + "packages/test-agent/tests/bridge.test.ts": 319, + "packages/test-agent/tests/command-lazy.test.ts": 5508, + "packages/test-agent/tests/components.test.ts": 102335, + "packages/test-agent/tests/controller.test.ts": 482, + "packages/test-agent/tests/cross-package-resolution.test.ts": 1781, + "packages/test-agent/tests/native-launch.test.ts": 54441, + "packages/test-agent/tests/net.test.ts": 380, + "packages/test-agent/tests/profile.test.ts": 750, + "packages/test-agent/tests/protocol.test.ts": 371, + "packages/test-agent/tests/provider.test.ts": 953, + "packages/test-agent/tests/public-api.test.ts": 1042, + "packages/test-agent/tests/route-slot.test.ts": 325, + "packages/test-agent/tests/session-configuration.test.ts": 149536, + "packages/test-agent/tests/smoke.test.ts": 25506, + "packages/test-agent/tests/template.test.ts": 749, + "packages/test-agent/tests/worker-lifecycle.test.ts": 25506, + "packages/test-support/tests/journal.test.ts": 337, + "packages/testing/tests/agent-composition.test.ts": 899, + "packages/testing/tests/assert-throws.test.ts": 1791, + "packages/testing/tests/assert.test.ts": 328, + "packages/testing/tests/assertions.test.ts": 2169, + "packages/testing/tests/boundary-contract.test.ts": 1445, + "packages/testing/tests/cli.test.ts": 12015, + "packages/testing/tests/execution-harness.test.ts": 4283, + "packages/testing/tests/replay.test.ts": 1716, + "packages/testing/tests/smoke.test.ts": 2283, + "packages/testing/tests/test-component.test.ts": 2296, + "packages/testing/tests/testing-activation.test.ts": 1933, + "packages/testing/tests/testing-boundary.test.ts": 1189, + "packages/testing/tests/testing-mode.test.ts": 2332, + "packages/testing/tests/use-testing.test.ts": 1565, + "packages/web/tests/assets.test.ts": 319, + "packages/web/tests/client-logic.test.ts": 347, + "packages/web/tests/compile.test.ts": 1120, + "packages/web/tests/component.test.ts": 1653, + "packages/web/tests/declaration.test.ts": 956, + "packages/web/tests/document.test.ts": 1081, + "packages/web/tests/elicitation.test.ts": 963, + "packages/web/tests/live-form.test.ts": 901, + "packages/web/tests/markdown.test.ts": 423, + "packages/web/tests/opener.test.ts": 868, + "packages/web/tests/page.test.ts": 326, + "packages/web/tests/responder.test.ts": 1065, + "packages/web/tests/server-lifecycle.test.ts": 993, + "packages/web/tests/server.test.ts": 1170, + "packages/workflow/tests/generated-observations.test.ts": 1100, + "packages/workflow/tests/retained-run.test.ts": 1318, + "packages/workflow/tests/service-denial.test.ts": 338, + "packages/workflow/tests/workflow-bundle.test.ts": 1023, + "packages/workflow/tests/workflow-definition.test.ts": 800, + "packages/workflow/tests/workflow-fork.test.ts": 738, + "packages/workflow/tests/workflow-run.test.ts": 1165, + "packages/workflow/tests/workspace-effect.test.ts": 1801, + "packages/workflow/tests/xmd-artifact.test.ts": 2217, + "scripts/tests/adapter-vendor.test.ts": 692, + "scripts/tests/bootstrap-npm-package.test.ts": 6394, + "scripts/tests/bump-version.test.ts": 387, + "scripts/tests/changed-paths.test.ts": 334, + "scripts/tests/ci-workflow.test.ts": 2703, + "scripts/tests/claude-preparation.test.ts": 946, + "scripts/tests/claude-snapshot-module.test.ts": 525, + "scripts/tests/consumer-cycle.test.ts": 430, + "scripts/tests/filesystem-contract-workflow.test.ts": 373, + "scripts/tests/main-green.test.ts": 541, + "scripts/tests/main-health.test.ts": 387, + "scripts/tests/no-module-scoped-registry.test.ts": 859, + "scripts/tests/no-redundant-test-scope.test.ts": 3505, + "scripts/tests/no-sync-filesystem.test.ts": 3750, + "scripts/tests/no-yield-in-finally.test.ts": 847, + "scripts/tests/oxlint-policy.test.ts": 8995, + "scripts/tests/packaged-document.test.ts": 388, + "scripts/tests/prefer-effection-operation.test.ts": 6731, + "scripts/tests/prefer-effection-result.test.ts": 3024, + "scripts/tests/publish-workflow-membership.test.ts": 397, + "scripts/tests/review-infrastructure.test.ts": 2430, + "scripts/tests/runtime-exclusions.test.ts": 637, + "scripts/tests/runtime-tests.test.ts": 363, + "scripts/tests/scope-bound-event-registration.test.ts": 3565, + "scripts/tests/staged-write.test.ts": 339, + "scripts/tests/test-file-discovery.test.ts": 425, + "scripts/tests/test-shards.test.ts": 355, + "scripts/tests/test-weights.test.ts": 560, + "scripts/tests/tracked.test.ts": 340, + "scripts/tests/verify-coordinator.test.ts": 486, + "scripts/tests/version-lockstep.test.ts": 421, + "scripts/tests/web-client-module.test.ts": 343, + "scripts/tests/workspace.test.ts": 335 }, "bun": { - "packages/acp/tests/acceptance.test.ts": 712, - "packages/acp/tests/acpx-checkpoint-meta.test.ts": 4839, - "packages/acp/tests/acpx-materialization.test.ts": 16077, - "packages/acp/tests/acpx-session-save.test.ts": 179, - "packages/acp/tests/acpx-transient-env.test.ts": 654, - "packages/acp/tests/adapter-materialization.test.ts": 75211, - "packages/acp/tests/adapter-protocol.test.ts": 37425, - "packages/acp/tests/agent-certification-report.test.ts": 52, - "packages/acp/tests/agent-certification.test.ts": 22245, - "packages/acp/tests/codex-surfaces.test.ts": 110, - "packages/acp/tests/host-conversation.test.ts": 5465, - "packages/acp/tests/native-launch.test.ts": 881, - "packages/acp/tests/permission-bridge.test.ts": 322, - "packages/acp/tests/provider.test.ts": 1012, - "packages/acp/tests/released-runtime.test.ts": 24046, - "packages/acp/tests/serial-queue.test.ts": 62, - "packages/acp/tests/session-key.test.ts": 133, - "packages/acp/tests/terminal-screen.test.ts": 76, - "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 12061, - "packages/cli/tests/adversarial-artifact-return.test.ts": 25881, - "packages/cli/tests/adversarial-delivery.test.ts": 34132, - "packages/cli/tests/adversarial-friction.test.ts": 91534, - "packages/cli/tests/adversarial-implementation.test.ts": 87497, - "packages/cli/tests/adversarial-launch-policy.test.ts": 64460, - "packages/cli/tests/adversarial-preparation.test.ts": 150526, - "packages/cli/tests/adversarial-production-role.test.ts": 47900, - "packages/cli/tests/adversarial-session-continuity.test.ts": 8603, - "packages/cli/tests/adversarial-workflow.test.ts": 70086, - "packages/cli/tests/agent-adapters.test.ts": 2079, - "packages/cli/tests/agent-certification.test.ts": 194635, - "packages/cli/tests/agent-cli.test.ts": 23436, + "packages/acp/tests/acceptance.test.ts": 740, + "packages/acp/tests/acpx-checkpoint-meta.test.ts": 4875, + "packages/acp/tests/acpx-materialization.test.ts": 15949, + "packages/acp/tests/acpx-session-save.test.ts": 151, + "packages/acp/tests/acpx-transient-env.test.ts": 660, + "packages/acp/tests/adapter-materialization.test.ts": 76718, + "packages/acp/tests/adapter-protocol.test.ts": 43383, + "packages/acp/tests/agent-certification-report.test.ts": 58, + "packages/acp/tests/agent-certification.test.ts": 22558, + "packages/acp/tests/codex-surfaces.test.ts": 118, + "packages/acp/tests/host-conversation.test.ts": 5598, + "packages/acp/tests/native-launch.test.ts": 916, + "packages/acp/tests/permission-bridge.test.ts": 332, + "packages/acp/tests/prompt-usage.test.ts": 8149, + "packages/acp/tests/provider.test.ts": 1074, + "packages/acp/tests/released-runtime.test.ts": 24883, + "packages/acp/tests/serial-queue.test.ts": 63, + "packages/acp/tests/session-key.test.ts": 137, + "packages/acp/tests/terminal-screen.test.ts": 80, + "packages/cli/tests/adversarial-artifact-return-recovery.test.ts": 13268, + "packages/cli/tests/adversarial-artifact-return.test.ts": 28695, + "packages/cli/tests/adversarial-delivery.test.ts": 37610, + "packages/cli/tests/adversarial-friction.test.ts": 98390, + "packages/cli/tests/adversarial-implementation.test.ts": 96028, + "packages/cli/tests/adversarial-launch-policy.test.ts": 69560, + "packages/cli/tests/adversarial-preparation.test.ts": 156876, + "packages/cli/tests/adversarial-production-role.test.ts": 50345, + "packages/cli/tests/adversarial-session-continuity.test.ts": 8899, + "packages/cli/tests/adversarial-workflow.test.ts": 77423, + "packages/cli/tests/agent-adapters.test.ts": 2255, + "packages/cli/tests/agent-certification.test.ts": 196326, + "packages/cli/tests/agent-cli.test.ts": 23486, "packages/cli/tests/agent-config.test.ts": 34, - "packages/cli/tests/agent-options-cli.test.ts": 10405, - "packages/cli/tests/agent-options.test.ts": 32, - "packages/cli/tests/agent-session-coordinator.test.ts": 384, - "packages/cli/tests/cli-help.test.ts": 49928, - "packages/cli/tests/command.test.ts": 10844, - "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 2758, - "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 112788, - "packages/cli/tests/document-suites/props/props-markdown.test.ts": 2190, - "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 4773, - "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 4469, - "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 8966, - "packages/cli/tests/evaluate-component.test.ts": 7462, - "packages/cli/tests/fetch-cli.test.ts": 2332, - "packages/cli/tests/file-stream.test.ts": 47, - "packages/cli/tests/inline-cli.test.ts": 31634, - "packages/cli/tests/launch-timeout.test.ts": 10074, - "packages/cli/tests/packaged-document.test.ts": 441, - "packages/cli/tests/plan-args.test.ts": 38, - "packages/cli/tests/plan-cli.test.ts": 62481, - "packages/cli/tests/plan-command-document.test.ts": 5044, - "packages/cli/tests/plan-component.test.ts": 8008, - "packages/cli/tests/plan-host-acts.test.ts": 992, - "packages/cli/tests/plan.test.ts": 10954, - "packages/cli/tests/plugin-cli.test.ts": 38880, - "packages/cli/tests/plugin-host.test.ts": 1783, - "packages/cli/tests/plugin-modules.test.ts": 1212, - "packages/cli/tests/plugin-selection.test.ts": 34, - "packages/cli/tests/process-retention.test.ts": 2481, - "packages/cli/tests/props-cli.test.ts": 29005, - "packages/cli/tests/props-schema.test.ts": 322, + "packages/cli/tests/agent-options-cli.test.ts": 10601, + "packages/cli/tests/agent-options.test.ts": 34, + "packages/cli/tests/agent-session-coordinator.test.ts": 408, + "packages/cli/tests/cli-help.test.ts": 50703, + "packages/cli/tests/command.test.ts": 10922, + "packages/cli/tests/document-suites/inline/inline-markdown.test.ts": 2860, + "packages/cli/tests/document-suites/plan/plan-markdown.test.ts": 115187, + "packages/cli/tests/document-suites/props/props-markdown.test.ts": 2266, + "packages/cli/tests/document-suites/syntax/syntax-markdown.test.ts": 4917, + "packages/cli/tests/document-suites/targets/targets-markdown.test.ts": 4631, + "packages/cli/tests/document-suites/verbose/verbose-markdown.test.ts": 9395, + "packages/cli/tests/evaluate-component.test.ts": 7997, + "packages/cli/tests/fetch-cli.test.ts": 2443, + "packages/cli/tests/file-stream.test.ts": 45, + "packages/cli/tests/inline-cli.test.ts": 33536, + "packages/cli/tests/launch-timeout.test.ts": 10084, + "packages/cli/tests/packaged-document.test.ts": 467, + "packages/cli/tests/parse-failure.test.ts": 3663, + "packages/cli/tests/plan-args.test.ts": 35, + "packages/cli/tests/plan-cli.test.ts": 67218, + "packages/cli/tests/plan-command-document.test.ts": 5572, + "packages/cli/tests/plan-component.test.ts": 8762, + "packages/cli/tests/plan-host-acts.test.ts": 1048, + "packages/cli/tests/plan.test.ts": 12383, + "packages/cli/tests/plugin-cli.test.ts": 41106, + "packages/cli/tests/plugin-host.test.ts": 1853, + "packages/cli/tests/plugin-modules.test.ts": 1307, + "packages/cli/tests/plugin-selection.test.ts": 36, + "packages/cli/tests/process-retention.test.ts": 2755, + "packages/cli/tests/props-cli.test.ts": 30329, + "packages/cli/tests/props-schema.test.ts": 338, "packages/cli/tests/props-sources.test.ts": 102, - "packages/cli/tests/protected-input-cli.test.ts": 7291, - "packages/cli/tests/protected-input-forms.test.ts": 924, - "packages/cli/tests/repl-admission.test.ts": 40, - "packages/cli/tests/repl-agent-execution.test.ts": 3841, - "packages/cli/tests/repl-agent-interface.test.ts": 39745, - "packages/cli/tests/repl-agent-journey.test.ts": 18500, - "packages/cli/tests/repl-asset-import.test.ts": 6614, - "packages/cli/tests/repl-boundaries.test.ts": 1280, - "packages/cli/tests/repl-caret.test.ts": 583, - "packages/cli/tests/repl-composition.test.ts": 2384, - "packages/cli/tests/repl-entries.test.ts": 8272, - "packages/cli/tests/repl-exec-output.test.ts": 4692, - "packages/cli/tests/repl-execution.test.ts": 1953, - "packages/cli/tests/repl-exit-pty.test.ts": 3743, - "packages/cli/tests/repl-field-ownership.test.ts": 588, - "packages/cli/tests/repl-focus-navigation.test.ts": 78, - "packages/cli/tests/repl-focus-regions.test.ts": 8849, - "packages/cli/tests/repl-forms.test.ts": 7162, - "packages/cli/tests/repl-git-observations.test.ts": 2246, - "packages/cli/tests/repl-headless-attach.test.ts": 37833, - "packages/cli/tests/repl-headless-clients.test.ts": 40443, - "packages/cli/tests/repl-headless-submit-process.test.ts": 4898, - "packages/cli/tests/repl-headless-submit.test.ts": 21977, - "packages/cli/tests/repl-headless.test.ts": 11485, - "packages/cli/tests/repl-history.test.ts": 2346, - "packages/cli/tests/repl-include-pty.test.ts": 24253, - "packages/cli/tests/repl-issues-pty.test.ts": 2594, - "packages/cli/tests/repl-journey.test.ts": 38720, - "packages/cli/tests/repl-layout-admission.test.ts": 7017, - "packages/cli/tests/repl-lifecycle.test.ts": 779, - "packages/cli/tests/repl-model.test.ts": 1828, - "packages/cli/tests/repl-mounts-pty.test.ts": 23431, - "packages/cli/tests/repl-page.test.ts": 3531, - "packages/cli/tests/repl-plugin.test.ts": 2770, - "packages/cli/tests/repl-presentation-style.test.ts": 29825, - "packages/cli/tests/repl-project-files.test.ts": 10057, - "packages/cli/tests/repl-project-recovery.test.ts": 6619, - "packages/cli/tests/repl-push-pty.test.ts": 5613, - "packages/cli/tests/repl-repositories-pty.test.ts": 7783, - "packages/cli/tests/repl-retained-history.test.ts": 890, - "packages/cli/tests/repl-route.test.ts": 1387, - "packages/cli/tests/repl-sidekick-acp.test.ts": 235116, - "packages/cli/tests/repl-sidekick-configuration.test.ts": 19311, - "packages/cli/tests/repl-sidekick-controls.test.ts": 36329, - "packages/cli/tests/repl-sidekick-dispatch.test.ts": 8939, - "packages/cli/tests/repl-sidekick-journey.test.ts": 30019, - "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 17264, + "packages/cli/tests/protected-input-cli.test.ts": 7445, + "packages/cli/tests/protected-input-forms.test.ts": 930, + "packages/cli/tests/repl-admission.test.ts": 38, + "packages/cli/tests/repl-agent-execution.test.ts": 4049, + "packages/cli/tests/repl-agent-interface.test.ts": 40068, + "packages/cli/tests/repl-agent-journey.test.ts": 19286, + "packages/cli/tests/repl-asset-import.test.ts": 6901, + "packages/cli/tests/repl-boundaries.test.ts": 1363, + "packages/cli/tests/repl-caret.test.ts": 601, + "packages/cli/tests/repl-composition.test.ts": 2518, + "packages/cli/tests/repl-entries.test.ts": 14902, + "packages/cli/tests/repl-exec-output.test.ts": 5368, + "packages/cli/tests/repl-execution.test.ts": 2070, + "packages/cli/tests/repl-exit-pty.test.ts": 3881, + "packages/cli/tests/repl-field-ownership.test.ts": 608, + "packages/cli/tests/repl-focus-navigation.test.ts": 68, + "packages/cli/tests/repl-focus-regions.test.ts": 10177, + "packages/cli/tests/repl-forms.test.ts": 7539, + "packages/cli/tests/repl-git-observations.test.ts": 2250, + "packages/cli/tests/repl-headless-attach.test.ts": 39924, + "packages/cli/tests/repl-headless-clients.test.ts": 41550, + "packages/cli/tests/repl-headless-submit-process.test.ts": 5146, + "packages/cli/tests/repl-headless-submit.test.ts": 23162, + "packages/cli/tests/repl-headless.test.ts": 11952, + "packages/cli/tests/repl-history.test.ts": 2389, + "packages/cli/tests/repl-include-pty.test.ts": 25701, + "packages/cli/tests/repl-issues-pty.test.ts": 2771, + "packages/cli/tests/repl-journey.test.ts": 44618, + "packages/cli/tests/repl-layout-admission.test.ts": 7931, + "packages/cli/tests/repl-lifecycle.test.ts": 830, + "packages/cli/tests/repl-model.test.ts": 1964, + "packages/cli/tests/repl-mounts-pty.test.ts": 23817, + "packages/cli/tests/repl-page.test.ts": 4279, + "packages/cli/tests/repl-parse-failure.test.ts": 514, + "packages/cli/tests/repl-plugin.test.ts": 2961, + "packages/cli/tests/repl-presentation-style.test.ts": 31500, + "packages/cli/tests/repl-project-files.test.ts": 11965, + "packages/cli/tests/repl-project-recovery.test.ts": 7998, + "packages/cli/tests/repl-push-pty.test.ts": 5935, + "packages/cli/tests/repl-repositories-pty.test.ts": 8152, + "packages/cli/tests/repl-retained-history.test.ts": 948, + "packages/cli/tests/repl-route.test.ts": 1462, + "packages/cli/tests/repl-sidekick-acp.test.ts": 245902, + "packages/cli/tests/repl-sidekick-configuration.test.ts": 22019, + "packages/cli/tests/repl-sidekick-controls.test.ts": 37785, + "packages/cli/tests/repl-sidekick-dispatch.test.ts": 10647, + "packages/cli/tests/repl-sidekick-journey.test.ts": 34079, + "packages/cli/tests/repl-sidekick-lifecycle.test.ts": 18745, "packages/cli/tests/repl-sidekick-output.test.ts": 55, - "packages/cli/tests/repl-sidekick-presentation.test.ts": 68354, - "packages/cli/tests/repl-sidekick-recovery.test.ts": 80539, - "packages/cli/tests/repl-sidekick-storage.test.ts": 494, - "packages/cli/tests/repl-sidekick-testing.test.ts": 19296, - "packages/cli/tests/repl-sidekick-typing.test.ts": 15739, - "packages/cli/tests/repl-source-reading.test.ts": 1953, - "packages/cli/tests/repl-terminal.test.ts": 13127, - "packages/cli/tests/repl-unfinished-entry.test.ts": 17787, - "packages/cli/tests/repl-wheel-frame.test.ts": 1350, - "packages/cli/tests/repl-wheel-input.test.ts": 617, - "packages/cli/tests/repl-wheel-migration.test.ts": 34270, - "packages/cli/tests/repl-wheel-scroll.test.ts": 27893, - "packages/cli/tests/retained-mount-files.test.ts": 1365, - "packages/cli/tests/retained-mount-records.test.ts": 963, - "packages/cli/tests/retained-repository-observations.test.ts": 4400, - "packages/cli/tests/retained-repository-pull-request-records.test.ts": 321, - "packages/cli/tests/retained-repository-push-records.test.ts": 317, - "packages/cli/tests/retained-repository-records.test.ts": 300, - "packages/cli/tests/run-composition.test.ts": 4535, - "packages/cli/tests/run-deadline.test.ts": 2365, - "packages/cli/tests/run-profile.test.ts": 599, - "packages/cli/tests/run-timeouts.test.ts": 11454, - "packages/cli/tests/secret-detection-cli.test.ts": 18103, - "packages/cli/tests/service-document.test.ts": 1304, - "packages/cli/tests/service-host.test.ts": 1166, - "packages/cli/tests/session-launch-cli.test.ts": 5950, - "packages/cli/tests/session-ownership-cli.test.ts": 2632, - "packages/cli/tests/stdin-cli.test.ts": 31449, - "packages/cli/tests/stdout-delivery.test.ts": 45, - "packages/cli/tests/syntax-cli.test.ts": 31298, - "packages/cli/tests/targets-cli.test.ts": 22376, - "packages/cli/tests/test-root-profile.test.ts": 3726, - "packages/cli/tests/test-target.test.ts": 35098, - "packages/cli/tests/testing-activation.test.ts": 471, - "packages/cli/tests/testing-execution-host.test.ts": 53156, - "packages/cli/tests/upgrade-cli.test.ts": 19069, - "packages/cli/tests/upgrade-command-document.test.ts": 3238, - "packages/cli/tests/upgrade-output.test.ts": 689, - "packages/cli/tests/value-root.test.ts": 15605, - "packages/cli/tests/verbose-component.test.ts": 697, - "packages/cli/tests/workflow-activation.test.ts": 29958, - "packages/cli/tests/workflow-declaration.test.ts": 427, - "packages/cli/tests/workflow-host.test.ts": 4575, - "packages/cli/tests/workspace-mounts.test.ts": 3285, - "packages/code-review-agent/tests/doctor.test.ts": 29, - "packages/code-review-agent/tests/parse-diagnostics.test.ts": 37, - "packages/code-review-agent/tests/parse-diff.test.ts": 37, - "packages/code-review-agent/tests/parse-doctor.test.ts": 254, - "packages/code-review-agent/tests/parse-oxlint.test.ts": 29, - "packages/code-review-agent/tests/policy.test.ts": 31, - "packages/core/tests/agent-api.test.ts": 35, - "packages/core/tests/agent-components.test.ts": 1018, - "packages/core/tests/agent-conversation.test.ts": 2984, - "packages/core/tests/agent-function-components.test.ts": 3404, - "packages/core/tests/agent-session-launch.test.ts": 2768, - "packages/core/tests/agent-session-setup.test.ts": 39, - "packages/core/tests/agent-session-use.test.ts": 36, - "packages/core/tests/agent-stack.test.ts": 579, - "packages/core/tests/all.test.ts": 2397, - "packages/core/tests/answer-identity.test.ts": 47, - "packages/core/tests/answers-component.test.ts": 1182, - "packages/core/tests/answers-expansion-recursion.test.ts": 251, - "packages/core/tests/capture-errors.test.ts": 463, - "packages/core/tests/capture-props.test.ts": 256, - "packages/core/tests/cli-journal.test.ts": 10201, - "packages/core/tests/code-block-component.test.ts": 1918, - "packages/core/tests/compiler-boundary.test.ts": 289, - "packages/core/tests/component-api.test.ts": 48, - "packages/core/tests/component-expansion.test.ts": 517, - "packages/core/tests/component-invocation-hooks.test.ts": 722, - "packages/core/tests/component-module-loader.test.ts": 658, - "packages/core/tests/component-registration.test.ts": 1041, - "packages/core/tests/component-returns.test.ts": 1576, - "packages/core/tests/component-schema-conformance.test.ts": 405, - "packages/core/tests/config-api.test.ts": 285, - "packages/core/tests/construct-error-observation.test.ts": 595, - "packages/core/tests/contextual-cwd.test.ts": 1720, - "packages/core/tests/daemon-integration.test.ts": 941, - "packages/core/tests/daemon.test.ts": 57, - "packages/core/tests/declared-markdown-component.test.ts": 1809, - "packages/core/tests/document-output-api.test.ts": 37, - "packages/core/tests/document-target-execution.test.ts": 2888, - "packages/core/tests/document-targets.test.ts": 317, - "packages/core/tests/document-validation.test.ts": 1699, - "packages/core/tests/documentation-index.test.ts": 126, - "packages/core/tests/each.test.ts": 370, - "packages/core/tests/elicit-component.test.ts": 930, - "packages/core/tests/ephemeral-service.test.ts": 693, - "packages/core/tests/eval-bindings.test.ts": 423, + "packages/cli/tests/repl-sidekick-presentation.test.ts": 97060, + "packages/cli/tests/repl-sidekick-recovery.test.ts": 86823, + "packages/cli/tests/repl-sidekick-storage.test.ts": 653, + "packages/cli/tests/repl-sidekick-testing.test.ts": 21802, + "packages/cli/tests/repl-sidekick-typing.test.ts": 17672, + "packages/cli/tests/repl-source-reading.test.ts": 1964, + "packages/cli/tests/repl-task-tabs.test.ts": 5043, + "packages/cli/tests/repl-terminal.test.ts": 15356, + "packages/cli/tests/repl-todo-context.test.ts": 2534, + "packages/cli/tests/repl-todo-journey.test.ts": 9732, + "packages/cli/tests/repl-todo-recovery.test.ts": 2547, + "packages/cli/tests/repl-todo.test.ts": 606, + "packages/cli/tests/repl-unfinished-entry.test.ts": 18818, + "packages/cli/tests/repl-wheel-frame.test.ts": 1376, + "packages/cli/tests/repl-wheel-input.test.ts": 633, + "packages/cli/tests/repl-wheel-migration.test.ts": 34205, + "packages/cli/tests/repl-wheel-scroll.test.ts": 29556, + "packages/cli/tests/retained-mount-files.test.ts": 1379, + "packages/cli/tests/retained-mount-records.test.ts": 1025, + "packages/cli/tests/retained-repository-observations.test.ts": 5315, + "packages/cli/tests/retained-repository-pull-request-records.test.ts": 342, + "packages/cli/tests/retained-repository-push-records.test.ts": 318, + "packages/cli/tests/retained-repository-records.test.ts": 297, + "packages/cli/tests/run-composition.test.ts": 4681, + "packages/cli/tests/run-deadline.test.ts": 2462, + "packages/cli/tests/run-profile.test.ts": 626, + "packages/cli/tests/run-timeouts.test.ts": 11606, + "packages/cli/tests/secret-detection-cli.test.ts": 18576, + "packages/cli/tests/service-document.test.ts": 1323, + "packages/cli/tests/service-host.test.ts": 1218, + "packages/cli/tests/session-launch-cli.test.ts": 6152, + "packages/cli/tests/session-ownership-cli.test.ts": 2842, + "packages/cli/tests/stdin-cli.test.ts": 32565, + "packages/cli/tests/stdout-delivery.test.ts": 46, + "packages/cli/tests/syntax-cli.test.ts": 32549, + "packages/cli/tests/targets-cli.test.ts": 23869, + "packages/cli/tests/test-root-profile.test.ts": 4031, + "packages/cli/tests/test-target.test.ts": 37870, + "packages/cli/tests/testing-activation.test.ts": 545, + "packages/cli/tests/testing-execution-host.test.ts": 58403, + "packages/cli/tests/upgrade-cli.test.ts": 20412, + "packages/cli/tests/upgrade-command-document.test.ts": 3740, + "packages/cli/tests/upgrade-output.test.ts": 767, + "packages/cli/tests/value-root.test.ts": 17464, + "packages/cli/tests/verbose-component.test.ts": 766, + "packages/cli/tests/workflow-activation.test.ts": 32458, + "packages/cli/tests/workflow-declaration.test.ts": 457, + "packages/cli/tests/workflow-host.test.ts": 5027, + "packages/cli/tests/workspace-mounts.test.ts": 3910, + "packages/code-review-agent/tests/doctor.test.ts": 35, + "packages/code-review-agent/tests/parse-diagnostics.test.ts": 42, + "packages/code-review-agent/tests/parse-diff.test.ts": 42, + "packages/code-review-agent/tests/parse-doctor.test.ts": 268, + "packages/code-review-agent/tests/parse-oxlint.test.ts": 31, + "packages/code-review-agent/tests/policy.test.ts": 32, + "packages/core/tests/agent-api.test.ts": 39, + "packages/core/tests/agent-components.test.ts": 1141, + "packages/core/tests/agent-conversation.test.ts": 3973, + "packages/core/tests/agent-function-components.test.ts": 3756, + "packages/core/tests/agent-session-launch.test.ts": 2922, + "packages/core/tests/agent-session-setup.test.ts": 43, + "packages/core/tests/agent-session-use.test.ts": 37, + "packages/core/tests/agent-stack.test.ts": 621, + "packages/core/tests/agent-usage.test.ts": 805, + "packages/core/tests/all.test.ts": 2400, + "packages/core/tests/answer-identity.test.ts": 50, + "packages/core/tests/answers-component.test.ts": 1259, + "packages/core/tests/answers-expansion-recursion.test.ts": 255, + "packages/core/tests/capture-errors.test.ts": 483, + "packages/core/tests/capture-props.test.ts": 272, + "packages/core/tests/cli-journal.test.ts": 10656, + "packages/core/tests/code-block-component.test.ts": 2026, + "packages/core/tests/compiler-boundary.test.ts": 299, + "packages/core/tests/component-api.test.ts": 45, + "packages/core/tests/component-expansion.test.ts": 544, + "packages/core/tests/component-invocation-hooks.test.ts": 765, + "packages/core/tests/component-module-loader.test.ts": 687, + "packages/core/tests/component-registration.test.ts": 1138, + "packages/core/tests/component-returns.test.ts": 1781, + "packages/core/tests/component-schema-conformance.test.ts": 437, + "packages/core/tests/config-api.test.ts": 323, + "packages/core/tests/construct-error-observation.test.ts": 627, + "packages/core/tests/contextual-cwd.test.ts": 1771, + "packages/core/tests/daemon-integration.test.ts": 987, + "packages/core/tests/daemon.test.ts": 61, + "packages/core/tests/declared-markdown-component.test.ts": 2013, + "packages/core/tests/document-output-api.test.ts": 40, + "packages/core/tests/document-target-execution.test.ts": 3263, + "packages/core/tests/document-targets.test.ts": 358, + "packages/core/tests/document-validation.test.ts": 1900, + "packages/core/tests/documentation-index.test.ts": 137, + "packages/core/tests/each.test.ts": 384, + "packages/core/tests/elicit-component.test.ts": 1043, + "packages/core/tests/ephemeral-service.test.ts": 750, + "packages/core/tests/eval-bindings.test.ts": 461, "packages/core/tests/eval-context.test.ts": 68, - "packages/core/tests/eval-durable.test.ts": 371, - "packages/core/tests/eval-error-mode.test.ts": 579, - "packages/core/tests/eval-interpolate.test.ts": 36, - "packages/core/tests/eval-middleware.test.ts": 80, - "packages/core/tests/eval-persist.test.ts": 420, - "packages/core/tests/eval-return.test.ts": 355, - "packages/core/tests/eval-scope.test.ts": 278, - "packages/core/tests/eval-timeout.test.ts": 385, - "packages/core/tests/eval-transform.test.ts": 56, - "packages/core/tests/evaluate-component.test.ts": 2616, - "packages/core/tests/evaluate-loaded-copy.test.ts": 450, - "packages/core/tests/evaluate-provider-lifetime.test.ts": 495, - "packages/core/tests/evaluation-profile.test.ts": 140, - "packages/core/tests/exec-timeout.test.ts": 412, - "packages/core/tests/execute.test.ts": 1098, - "packages/core/tests/execution-init.test.ts": 496, - "packages/core/tests/execution-protocol.test.ts": 1419, - "packages/core/tests/expand.test.ts": 1577, - "packages/core/tests/expansion-identity.test.ts": 1042, - "packages/core/tests/expansion-metadata.test.ts": 276, - "packages/core/tests/expression-props.test.ts": 742, - "packages/core/tests/fail-component.test.ts": 630, - "packages/core/tests/failure-printing.test.ts": 561, - "packages/core/tests/fatal-cause.test.ts": 150, - "packages/core/tests/fetch-component.test.ts": 1486, - "packages/core/tests/file-component.test.ts": 2205, - "packages/core/tests/file-delete-component.test.ts": 528, - "packages/core/tests/files-fatal.test.ts": 1315, - "packages/core/tests/filesystem-placement.test.ts": 602, - "packages/core/tests/fixed-filesystem-components.test.ts": 681, - "packages/core/tests/foreground-exec.test.ts": 1057, - "packages/core/tests/frontmatter.test.ts": 47, - "packages/core/tests/function-components.test.ts": 1140, - "packages/core/tests/generated-composition.test.ts": 1009, - "packages/core/tests/generated-owner-metadata.test.ts": 434, - "packages/core/tests/generated-selection.test.ts": 785, - "packages/core/tests/generated-xmd-preview.test.ts": 197, - "packages/core/tests/generated-xmd-root.test.ts": 1376, - "packages/core/tests/generated-xmd.test.ts": 1730, - "packages/core/tests/glob-api.test.ts": 76, - "packages/core/tests/glob-component.test.ts": 1078, - "packages/core/tests/guarded-journal.test.ts": 408, - "packages/core/tests/has-content.test.ts": 224, - "packages/core/tests/heal.test.ts": 209, - "packages/core/tests/host-generated-root.test.ts": 1082, - "packages/core/tests/if.test.ts": 715, - "packages/core/tests/inline-root.test.ts": 421, - "packages/core/tests/invocation-failures.test.ts": 106, - "packages/core/tests/invocation-identity.test.ts": 1108, - "packages/core/tests/invocation-scope.test.ts": 479, - "packages/core/tests/journal-source-position.test.ts": 478, - "packages/core/tests/json-component.test.ts": 2055, - "packages/core/tests/json.test.ts": 35, - "packages/core/tests/let.test.ts": 611, - "packages/core/tests/loop.test.ts": 1106, - "packages/core/tests/named-slots.test.ts": 1059, - "packages/core/tests/output-error-mode.test.ts": 1463, - "packages/core/tests/output-normalize.test.ts": 40, - "packages/core/tests/output-terminal.test.ts": 131, - "packages/core/tests/parse-components.test.ts": 512, - "packages/core/tests/plan-response.test.ts": 66, - "packages/core/tests/plugin-api.test.ts": 178, - "packages/core/tests/plugin.test.ts": 174, - "packages/core/tests/props-binding.test.ts": 454, - "packages/core/tests/protected-content.test.ts": 41, - "packages/core/tests/protected-input-replay.test.ts": 798, - "packages/core/tests/protected-input.test.ts": 1139, - "packages/core/tests/query-component.test.ts": 589, - "packages/core/tests/registered-printed-errors.test.ts": 164, - "packages/core/tests/replay-stream.test.ts": 32, - "packages/core/tests/retain.test.ts": 402, - "packages/core/tests/root-composition.test.ts": 483, - "packages/core/tests/root-props.test.ts": 510, - "packages/core/tests/root-provider.test.ts": 467, - "packages/core/tests/sample-component.test.ts": 989, - "packages/core/tests/scanner.test.ts": 69, - "packages/core/tests/scope-local.test.ts": 33, - "packages/core/tests/secret-detection.test.ts": 995, - "packages/core/tests/secret-files.test.ts": 91, - "packages/core/tests/secret-rules.test.ts": 120, - "packages/core/tests/secret-scanner-baseline.test.ts": 125, - "packages/core/tests/secret-scanner.test.ts": 81, - "packages/core/tests/shared-operation-permissions.test.ts": 1343, - "packages/core/tests/slice-component.test.ts": 1171, - "packages/core/tests/source-inspection.test.ts": 60, - "packages/core/tests/source-position.test.ts": 292, - "packages/core/tests/state-ownership.test.ts": 345, - "packages/core/tests/streaming-emission.test.ts": 274, - "packages/core/tests/switch.test.ts": 731, - "packages/core/tests/syntax-api.test.ts": 511, - "packages/core/tests/syntax-catalog.test.ts": 711, - "packages/core/tests/syntax-component.test.ts": 1537, - "packages/core/tests/temp-dir.test.ts": 2686, + "packages/core/tests/eval-durable.test.ts": 397, + "packages/core/tests/eval-error-mode.test.ts": 627, + "packages/core/tests/eval-interpolate.test.ts": 41, + "packages/core/tests/eval-middleware.test.ts": 86, + "packages/core/tests/eval-persist.test.ts": 454, + "packages/core/tests/eval-return.test.ts": 390, + "packages/core/tests/eval-scope.test.ts": 306, + "packages/core/tests/eval-timeout.test.ts": 390, + "packages/core/tests/eval-transform.test.ts": 59, + "packages/core/tests/evaluate-component.test.ts": 2783, + "packages/core/tests/evaluate-loaded-copy.test.ts": 470, + "packages/core/tests/evaluate-provider-lifetime.test.ts": 546, + "packages/core/tests/evaluation-profile.test.ts": 145, + "packages/core/tests/exec-timeout.test.ts": 446, + "packages/core/tests/execute.test.ts": 1175, + "packages/core/tests/execution-init.test.ts": 528, + "packages/core/tests/execution-protocol.test.ts": 1727, + "packages/core/tests/expand.test.ts": 1748, + "packages/core/tests/expansion-identity.test.ts": 1188, + "packages/core/tests/expansion-metadata.test.ts": 297, + "packages/core/tests/expression-props.test.ts": 848, + "packages/core/tests/fail-component.test.ts": 700, + "packages/core/tests/failure-printing.test.ts": 609, + "packages/core/tests/fatal-cause.test.ts": 156, + "packages/core/tests/fetch-component.test.ts": 1646, + "packages/core/tests/file-component.test.ts": 2348, + "packages/core/tests/file-delete-component.test.ts": 579, + "packages/core/tests/files-fatal.test.ts": 1437, + "packages/core/tests/filesystem-placement.test.ts": 625, + "packages/core/tests/fixed-filesystem-components.test.ts": 753, + "packages/core/tests/foreground-exec.test.ts": 1203, + "packages/core/tests/frontmatter.test.ts": 46, + "packages/core/tests/function-components.test.ts": 1166, + "packages/core/tests/generated-composition.test.ts": 1138, + "packages/core/tests/generated-owner-metadata.test.ts": 481, + "packages/core/tests/generated-selection.test.ts": 859, + "packages/core/tests/generated-xmd-preview.test.ts": 206, + "packages/core/tests/generated-xmd-root.test.ts": 1513, + "packages/core/tests/generated-xmd.test.ts": 1952, + "packages/core/tests/glob-api.test.ts": 81, + "packages/core/tests/glob-component.test.ts": 1173, + "packages/core/tests/guarded-journal.test.ts": 428, + "packages/core/tests/has-content.test.ts": 223, + "packages/core/tests/heal.test.ts": 215, + "packages/core/tests/host-generated-root.test.ts": 1231, + "packages/core/tests/if.test.ts": 748, + "packages/core/tests/inline-root.test.ts": 469, + "packages/core/tests/invocation-failures.test.ts": 93, + "packages/core/tests/invocation-identity.test.ts": 1172, + "packages/core/tests/invocation-scope.test.ts": 497, + "packages/core/tests/journal-source-position.test.ts": 503, + "packages/core/tests/json-component.test.ts": 2089, + "packages/core/tests/json.test.ts": 36, + "packages/core/tests/let.test.ts": 684, + "packages/core/tests/loop.test.ts": 1158, + "packages/core/tests/named-slots.test.ts": 1170, + "packages/core/tests/output-error-mode.test.ts": 1567, + "packages/core/tests/output-normalize.test.ts": 43, + "packages/core/tests/output-terminal.test.ts": 136, + "packages/core/tests/parse-components.test.ts": 746, + "packages/core/tests/plan-response.test.ts": 70, + "packages/core/tests/plugin-api.test.ts": 189, + "packages/core/tests/plugin.test.ts": 181, + "packages/core/tests/props-binding.test.ts": 509, + "packages/core/tests/protected-content.test.ts": 62, + "packages/core/tests/protected-input-replay.test.ts": 861, + "packages/core/tests/protected-input.test.ts": 1263, + "packages/core/tests/query-component.test.ts": 661, + "packages/core/tests/registered-printed-errors.test.ts": 177, + "packages/core/tests/replay-stream.test.ts": 35, + "packages/core/tests/retain.test.ts": 428, + "packages/core/tests/root-composition.test.ts": 499, + "packages/core/tests/root-props.test.ts": 574, + "packages/core/tests/root-provider.test.ts": 504, + "packages/core/tests/sample-component.test.ts": 1087, + "packages/core/tests/scanner.test.ts": 68, + "packages/core/tests/scope-local.test.ts": 34, + "packages/core/tests/secret-detection.test.ts": 1064, + "packages/core/tests/secret-files.test.ts": 115, + "packages/core/tests/secret-rules.test.ts": 126, + "packages/core/tests/secret-scanner-baseline.test.ts": 122, + "packages/core/tests/secret-scanner.test.ts": 85, + "packages/core/tests/shared-operation-permissions.test.ts": 1459, + "packages/core/tests/slice-component.test.ts": 1295, + "packages/core/tests/source-inspection.test.ts": 62, + "packages/core/tests/source-position.test.ts": 310, + "packages/core/tests/state-ownership.test.ts": 371, + "packages/core/tests/streaming-emission.test.ts": 292, + "packages/core/tests/switch.test.ts": 781, + "packages/core/tests/syntax-api.test.ts": 555, + "packages/core/tests/syntax-catalog.test.ts": 781, + "packages/core/tests/syntax-component.test.ts": 1746, + "packages/core/tests/temp-dir.test.ts": 2800, "packages/core/tests/temp-file-compiler.test.ts": 64, - "packages/core/tests/text-interpolation.test.ts": 191, - "packages/core/tests/try-content.test.ts": 220, - "packages/core/tests/typescript-materialization.test.ts": 1170, - "packages/core/tests/unused-in-diff.test.ts": 422, - "packages/core/tests/validate.test.ts": 228, - "packages/core/tests/validation-integration.test.ts": 324, - "packages/core/tests/workflow-component-bundle.test.ts": 604, - "packages/core/tests/yaml-component.test.ts": 274, - "packages/durable-streams/tests/context.test.ts": 38, + "packages/core/tests/text-interpolation.test.ts": 198, + "packages/core/tests/try-content.test.ts": 233, + "packages/core/tests/typescript-materialization.test.ts": 1235, + "packages/core/tests/unused-in-diff.test.ts": 441, + "packages/core/tests/validate.test.ts": 267, + "packages/core/tests/validation-integration.test.ts": 349, + "packages/core/tests/workflow-component-bundle.test.ts": 678, + "packages/core/tests/yaml-component.test.ts": 299, + "packages/durable-streams/tests/context.test.ts": 37, "packages/durable-streams/tests/deterministic-id.test.ts": 60, - "packages/durable-streams/tests/divergence-api.test.ts": 48, - "packages/durable-streams/tests/divergence.test.ts": 58, - "packages/durable-streams/tests/durable-each.test.ts": 60, - "packages/durable-streams/tests/durable-position.test.ts": 49, - "packages/durable-streams/tests/durable-run.test.ts": 66, - "packages/durable-streams/tests/ephemeral.test.ts": 55, - "packages/durable-streams/tests/fail-stop.test.ts": 69, - "packages/durable-streams/tests/guard-stream.test.ts": 88, + "packages/durable-streams/tests/divergence-api.test.ts": 50, + "packages/durable-streams/tests/divergence.test.ts": 59, + "packages/durable-streams/tests/durable-each.test.ts": 65, + "packages/durable-streams/tests/durable-position.test.ts": 51, + "packages/durable-streams/tests/durable-run.test.ts": 70, + "packages/durable-streams/tests/ephemeral.test.ts": 58, + "packages/durable-streams/tests/fail-stop.test.ts": 71, + "packages/durable-streams/tests/guard-stream.test.ts": 90, "packages/durable-streams/tests/live-coordinator.test.ts": 57, - "packages/durable-streams/tests/parse.test.ts": 55, - "packages/durable-streams/tests/replay-guard.test.ts": 74, - "packages/durable-streams/tests/replay-index.test.ts": 41, - "packages/durable-streams/tests/retained.test.ts": 36, - "packages/durable-streams/tests/serialize-event.test.ts": 38, - "packages/durable-streams/tests/smoke.test.ts": 35, - "packages/durable-streams/tests/structured-concurrency.test.ts": 75, - "packages/durable-streams/tests/terminal-boundary.test.ts": 71, - "packages/durable-streams/tests/types.test.ts": 28, - "packages/git/tests/api-entrypoint.test.ts": 263, - "packages/git/tests/composition-component-binding.test.ts": 932, - "packages/git/tests/git-commit-read.test.ts": 7586, - "packages/git/tests/git-host-effect.test.ts": 6908, - "packages/git/tests/git-host-owned.test.ts": 1720, - "packages/git/tests/git-observation-records.test.ts": 309, - "packages/git/tests/git-observations.test.ts": 14621, - "packages/git/tests/git.test.ts": 303, - "packages/git/tests/github-activation.test.ts": 630, - "packages/git/tests/github-issues.test.ts": 291, - "packages/git/tests/github-pull-requests.test.ts": 283, - "packages/git/tests/issue-markdown.test.ts": 7422, - "packages/git/tests/issue-owned.test.ts": 2488, - "packages/git/tests/issue-records.test.ts": 255, - "packages/git/tests/module-partition.test.ts": 1503, - "packages/git/tests/plugin.test.ts": 709, - "packages/git/tests/provider-neutrality.test.ts": 885, - "packages/git/tests/pull-request-records.test.ts": 271, - "packages/git/tests/run-composition-lazy.test.ts": 747, - "packages/git/tests/selection-authentication.test.ts": 38, + "packages/durable-streams/tests/parse.test.ts": 44, + "packages/durable-streams/tests/replay-guard.test.ts": 67, + "packages/durable-streams/tests/replay-index.test.ts": 42, + "packages/durable-streams/tests/retained.test.ts": 43, + "packages/durable-streams/tests/serialize-event.test.ts": 40, + "packages/durable-streams/tests/smoke.test.ts": 56, + "packages/durable-streams/tests/structured-concurrency.test.ts": 89, + "packages/durable-streams/tests/terminal-boundary.test.ts": 74, + "packages/durable-streams/tests/types.test.ts": 33, + "packages/git/tests/api-entrypoint.test.ts": 275, + "packages/git/tests/composition-component-binding.test.ts": 1092, + "packages/git/tests/git-commit-read.test.ts": 8814, + "packages/git/tests/git-host-effect.test.ts": 7722, + "packages/git/tests/git-host-owned.test.ts": 1173, + "packages/git/tests/git-observation-records.test.ts": 278, + "packages/git/tests/git-observations.test.ts": 16217, + "packages/git/tests/git.test.ts": 327, + "packages/git/tests/github-activation.test.ts": 709, + "packages/git/tests/github-issues.test.ts": 313, + "packages/git/tests/github-pull-requests.test.ts": 298, + "packages/git/tests/issue-markdown.test.ts": 7951, + "packages/git/tests/issue-owned.test.ts": 2780, + "packages/git/tests/issue-records.test.ts": 279, + "packages/git/tests/module-partition.test.ts": 1558, + "packages/git/tests/plugin.test.ts": 829, + "packages/git/tests/provider-neutrality.test.ts": 938, + "packages/git/tests/pull-request-records.test.ts": 288, + "packages/git/tests/run-composition-lazy.test.ts": 958, + "packages/git/tests/selection-authentication.test.ts": 33, "packages/runtime/tests/duration.test.ts": 31, - "packages/runtime/tests/fetch.test.ts": 592, - "packages/runtime/tests/files-call.test.ts": 35, - "packages/runtime/tests/fs.test.ts": 58, - "packages/runtime/tests/host-files.test.ts": 707, - "packages/runtime/tests/native-launcher.test.ts": 2377, - "packages/runtime/tests/permissions.test.ts": 62, - "packages/runtime/tests/process.test.ts": 83, - "packages/runtime/tests/service.test.ts": 51, - "packages/test-agent/tests/acp-server.test.ts": 83, - "packages/test-agent/tests/behavior-engine.test.ts": 607, - "packages/test-agent/tests/bridge.test.ts": 69, - "packages/test-agent/tests/command-lazy.test.ts": 3403, - "packages/test-agent/tests/components.test.ts": 61437, - "packages/test-agent/tests/controller.test.ts": 170, - "packages/test-agent/tests/cross-package-resolution.test.ts": 1507, - "packages/test-agent/tests/native-launch.test.ts": 30413, - "packages/test-agent/tests/net.test.ts": 79, - "packages/test-agent/tests/profile.test.ts": 254, + "packages/runtime/tests/fetch.test.ts": 600, + "packages/runtime/tests/files-call.test.ts": 38, + "packages/runtime/tests/fs.test.ts": 59, + "packages/runtime/tests/host-files.test.ts": 730, + "packages/runtime/tests/native-launcher.test.ts": 2396, + "packages/runtime/tests/permissions.test.ts": 59, + "packages/runtime/tests/process.test.ts": 73, + "packages/runtime/tests/service.test.ts": 57, + "packages/test-agent/tests/acp-server.test.ts": 86, + "packages/test-agent/tests/behavior-engine.test.ts": 874, + "packages/test-agent/tests/bridge.test.ts": 66, + "packages/test-agent/tests/command-lazy.test.ts": 3575, + "packages/test-agent/tests/components.test.ts": 64533, + "packages/test-agent/tests/controller.test.ts": 178, + "packages/test-agent/tests/cross-package-resolution.test.ts": 1533, + "packages/test-agent/tests/native-launch.test.ts": 32166, + "packages/test-agent/tests/net.test.ts": 87, + "packages/test-agent/tests/profile.test.ts": 273, "packages/test-agent/tests/protocol.test.ts": 57, - "packages/test-agent/tests/provider.test.ts": 359, - "packages/test-agent/tests/public-api.test.ts": 406, - "packages/test-agent/tests/route-slot.test.ts": 55, - "packages/test-agent/tests/session-configuration.test.ts": 95760, - "packages/test-agent/tests/smoke.test.ts": 16646, - "packages/test-agent/tests/template.test.ts": 248, - "packages/test-agent/tests/worker-lifecycle.test.ts": 12399, - "packages/test-support/tests/journal.test.ts": 41, - "packages/testing/tests/agent-composition.test.ts": 493, - "packages/testing/tests/assert-throws.test.ts": 1803, - "packages/testing/tests/assert.test.ts": 51, - "packages/testing/tests/assertions.test.ts": 2451, - "packages/testing/tests/boundary-contract.test.ts": 1276, - "packages/testing/tests/cli.test.ts": 6975, - "packages/testing/tests/execution-harness.test.ts": 5573, - "packages/testing/tests/replay.test.ts": 1779, - "packages/testing/tests/smoke.test.ts": 1859, - "packages/testing/tests/test-component.test.ts": 2469, - "packages/testing/tests/testing-activation.test.ts": 2155, - "packages/testing/tests/testing-boundary.test.ts": 897, - "packages/testing/tests/testing-mode.test.ts": 2655, - "packages/testing/tests/use-testing.test.ts": 1429, - "packages/web/tests/assets.test.ts": 35, - "packages/web/tests/client-logic.test.ts": 40, - "packages/web/tests/compile.test.ts": 713, - "packages/web/tests/component.test.ts": 1327, - "packages/web/tests/declaration.test.ts": 520, - "packages/web/tests/document.test.ts": 643, - "packages/web/tests/elicitation.test.ts": 458, - "packages/web/tests/live-form.test.ts": 390, - "packages/web/tests/markdown.test.ts": 106, - "packages/web/tests/opener.test.ts": 386, - "packages/web/tests/page.test.ts": 32, - "packages/web/tests/responder.test.ts": 631, - "packages/web/tests/server-lifecycle.test.ts": 571, - "packages/web/tests/server.test.ts": 828, - "packages/workflow/tests/generated-observations.test.ts": 738, - "packages/workflow/tests/retained-run.test.ts": 1048, - "packages/workflow/tests/service-denial.test.ts": 47, - "packages/workflow/tests/workflow-bundle.test.ts": 636, - "packages/workflow/tests/workflow-definition.test.ts": 286, - "packages/workflow/tests/workflow-fork.test.ts": 261, - "packages/workflow/tests/workflow-run.test.ts": 826, - "packages/workflow/tests/workspace-effect.test.ts": 1139, - "scripts/tests/adapter-vendor.test.ts": 195, - "scripts/tests/bootstrap-npm-package.test.ts": 5499, - "scripts/tests/bump-version.test.ts": 62, + "packages/test-agent/tests/provider.test.ts": 376, + "packages/test-agent/tests/public-api.test.ts": 425, + "packages/test-agent/tests/route-slot.test.ts": 60, + "packages/test-agent/tests/session-configuration.test.ts": 101102, + "packages/test-agent/tests/smoke.test.ts": 17702, + "packages/test-agent/tests/template.test.ts": 274, + "packages/test-agent/tests/worker-lifecycle.test.ts": 14021, + "packages/test-support/tests/journal.test.ts": 40, + "packages/testing/tests/agent-composition.test.ts": 555, + "packages/testing/tests/assert-throws.test.ts": 2217, + "packages/testing/tests/assert.test.ts": 53, + "packages/testing/tests/assertions.test.ts": 2915, + "packages/testing/tests/boundary-contract.test.ts": 1546, + "packages/testing/tests/cli.test.ts": 7806, + "packages/testing/tests/execution-harness.test.ts": 7103, + "packages/testing/tests/replay.test.ts": 2132, + "packages/testing/tests/smoke.test.ts": 2190, + "packages/testing/tests/test-component.test.ts": 3165, + "packages/testing/tests/testing-activation.test.ts": 2599, + "packages/testing/tests/testing-boundary.test.ts": 1053, + "packages/testing/tests/testing-mode.test.ts": 3235, + "packages/testing/tests/use-testing.test.ts": 1807, + "packages/web/tests/assets.test.ts": 30, + "packages/web/tests/client-logic.test.ts": 38, + "packages/web/tests/compile.test.ts": 793, + "packages/web/tests/component.test.ts": 1484, + "packages/web/tests/declaration.test.ts": 593, + "packages/web/tests/document.test.ts": 698, + "packages/web/tests/elicitation.test.ts": 498, + "packages/web/tests/live-form.test.ts": 414, + "packages/web/tests/markdown.test.ts": 112, + "packages/web/tests/opener.test.ts": 416, + "packages/web/tests/page.test.ts": 47, + "packages/web/tests/responder.test.ts": 736, + "packages/web/tests/server-lifecycle.test.ts": 613, + "packages/web/tests/server.test.ts": 863, + "packages/workflow/tests/generated-observations.test.ts": 782, + "packages/workflow/tests/retained-run.test.ts": 1184, + "packages/workflow/tests/service-denial.test.ts": 49, + "packages/workflow/tests/workflow-bundle.test.ts": 682, + "packages/workflow/tests/workflow-definition.test.ts": 303, + "packages/workflow/tests/workflow-fork.test.ts": 278, + "packages/workflow/tests/workflow-run.test.ts": 930, + "packages/workflow/tests/workspace-effect.test.ts": 1162, + "scripts/tests/adapter-vendor.test.ts": 207, + "scripts/tests/bootstrap-npm-package.test.ts": 6080, + "scripts/tests/bump-version.test.ts": 69, "scripts/tests/changed-paths.test.ts": 36, - "scripts/tests/ci-workflow.test.ts": 1771, - "scripts/tests/claude-preparation.test.ts": 690, - "scripts/tests/claude-snapshot-module.test.ts": 101, - "scripts/tests/consumer-cycle.test.ts": 80, - "scripts/tests/filesystem-contract-workflow.test.ts": 57, - "scripts/tests/main-green.test.ts": 270, - "scripts/tests/main-health.test.ts": 80, - "scripts/tests/no-module-scoped-registry.test.ts": 805, - "scripts/tests/no-redundant-test-scope.test.ts": 4337, - "scripts/tests/no-sync-filesystem.test.ts": 4133, - "scripts/tests/no-yield-in-finally.test.ts": 772, - "scripts/tests/oxlint-policy.test.ts": 11582, - "scripts/tests/packaged-document.test.ts": 68, - "scripts/tests/prefer-effection-operation.test.ts": 8795, - "scripts/tests/prefer-effection-result.test.ts": 3706, - "scripts/tests/publish-workflow-membership.test.ts": 75, - "scripts/tests/review-infrastructure.test.ts": 1997, - "scripts/tests/runtime-exclusions.test.ts": 293, - "scripts/tests/runtime-tests.test.ts": 64, - "scripts/tests/scope-bound-event-registration.test.ts": 4055, - "scripts/tests/staged-write.test.ts": 56, - "scripts/tests/test-file-discovery.test.ts": 165, - "scripts/tests/test-shards.test.ts": 40, - "scripts/tests/test-weights.test.ts": 253, - "scripts/tests/tracked.test.ts": 39, - "scripts/tests/verify-coordinator.test.ts": 209, - "scripts/tests/version-lockstep.test.ts": 95, - "scripts/tests/web-client-module.test.ts": 48, - "scripts/tests/workspace.test.ts": 45 + "scripts/tests/ci-workflow.test.ts": 1972, + "scripts/tests/claude-preparation.test.ts": 717, + "scripts/tests/claude-snapshot-module.test.ts": 105, + "scripts/tests/consumer-cycle.test.ts": 87, + "scripts/tests/filesystem-contract-workflow.test.ts": 56, + "scripts/tests/main-green.test.ts": 281, + "scripts/tests/main-health.test.ts": 101, + "scripts/tests/no-module-scoped-registry.test.ts": 832, + "scripts/tests/no-redundant-test-scope.test.ts": 4539, + "scripts/tests/no-sync-filesystem.test.ts": 4379, + "scripts/tests/no-yield-in-finally.test.ts": 827, + "scripts/tests/oxlint-policy.test.ts": 12058, + "scripts/tests/packaged-document.test.ts": 74, + "scripts/tests/prefer-effection-operation.test.ts": 9569, + "scripts/tests/prefer-effection-result.test.ts": 3802, + "scripts/tests/publish-workflow-membership.test.ts": 77, + "scripts/tests/review-infrastructure.test.ts": 2160, + "scripts/tests/runtime-exclusions.test.ts": 298, + "scripts/tests/runtime-tests.test.ts": 68, + "scripts/tests/scope-bound-event-registration.test.ts": 4178, + "scripts/tests/staged-write.test.ts": 59, + "scripts/tests/test-file-discovery.test.ts": 179, + "scripts/tests/test-shards.test.ts": 72, + "scripts/tests/test-weights.test.ts": 281, + "scripts/tests/tracked.test.ts": 40, + "scripts/tests/verify-coordinator.test.ts": 214, + "scripts/tests/version-lockstep.test.ts": 91, + "scripts/tests/web-client-module.test.ts": 47, + "scripts/tests/workspace.test.ts": 44 } } }