From ea57d86ab3fa9225e41d806b3bdea3b117837c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:54:32 +0100 Subject: [PATCH 1/9] refactor(test): colocate Maestro fuzz coverage with parser --- ...am-ir-command-parser-fuzz-coverage.test.ts | 33 +++++++++++++++++++ packages/test-kit/src/index.ts | 9 +++++ .../src/maestro-validation-arbitraries.ts | 3 +- .../test-kit/src}/validation-values.ts | 12 +++++++ scripts/fuzz/envelope.ts | 5 +-- scripts/fuzz/validation-arbitraries-cli.ts | 2 +- .../validation-arbitraries-maestro.test.ts | 30 +---------------- scripts/fuzz/validation-arbitraries.ts | 2 +- scripts/fuzz/validation-case.ts | 20 ++++------- 9 files changed, 67 insertions(+), 49 deletions(-) create mode 100644 packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts rename scripts/fuzz/validation-arbitraries-maestro.ts => packages/test-kit/src/maestro-validation-arbitraries.ts (98%) rename {scripts/fuzz => packages/test-kit/src}/validation-values.ts (66%) diff --git a/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts new file mode 100644 index 0000000000..4ed0ec8395 --- /dev/null +++ b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts @@ -0,0 +1,33 @@ +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { maestroValidationArb } from '@agent-device/test-kit'; +import { SUPPORTED_MAESTRO_COMMAND_NAMES } from './program-ir-command-parser.ts'; + +const sample = fc.sample(maestroValidationArb, { numRuns: 3_000, seed: 1 }); + +describe('shape coverage against the converter', () => { + // The valid Maestro shapes are rendered by hand here, so the one thing that cannot be left to + // drift is which commands they cover: a command the converter accepts but this generator never + // emits is a hole the nightly cannot see. Waivers are explicit and must say why. + const NOT_GENERATED = new Set([ + // Both take a nested command list whose bodies the repeat/runFlow cases already cover. + 'retry', + // Needs a JS file on disk to resolve, which a generated case has no way to provide. + 'runScript', + ]); + + it('emits every command the converter supports, or waives it explicitly', () => { + const emitted = new Set( + sample.flatMap((entry) => { + const { payload } = JSON.parse(entry) as { payload: string | string[] }; + return typeof payload === 'string' ? payload.split('\n') : []; + }), + ); + const covered = (command: string) => + [...emitted].some((line) => line.trim().startsWith(`- ${command}`)); + const missing = SUPPORTED_MAESTRO_COMMAND_NAMES.filter( + (command) => !NOT_GENERATED.has(command) && !covered(command), + ); + expect(missing).toEqual([]); + }); +}); diff --git a/packages/test-kit/src/index.ts b/packages/test-kit/src/index.ts index 6a78bd476c..02cb6ff37c 100644 --- a/packages/test-kit/src/index.ts +++ b/packages/test-kit/src/index.ts @@ -177,3 +177,12 @@ export { type StreamCapture, type StreamCaptureOptions, } from './stream-capture.ts'; + +export { maestroValidationArb, MAESTRO_MUTATION_NAMES } from './maestro-validation-arbitraries.ts'; +export { + ACCEPT, + SAFE_VALUES, + encodeValidationCase, + type ValidationCase, + type ValidationExpectation, +} from './validation-values.ts'; diff --git a/scripts/fuzz/validation-arbitraries-maestro.ts b/packages/test-kit/src/maestro-validation-arbitraries.ts similarity index 98% rename from scripts/fuzz/validation-arbitraries-maestro.ts rename to packages/test-kit/src/maestro-validation-arbitraries.ts index 6b47745bb7..57b593c873 100644 --- a/scripts/fuzz/validation-arbitraries-maestro.ts +++ b/packages/test-kit/src/maestro-validation-arbitraries.ts @@ -7,8 +7,7 @@ // cannot silently drift from the converter. import fc from 'fast-check'; -import { encodeValidationCase } from './validation-case.ts'; -import { ACCEPT, SAFE_VALUES } from './validation-values.ts'; +import { ACCEPT, SAFE_VALUES, encodeValidationCase } from './validation-values.ts'; // --------------------------------------------------------------------------------------------- diff --git a/scripts/fuzz/validation-values.ts b/packages/test-kit/src/validation-values.ts similarity index 66% rename from scripts/fuzz/validation-values.ts rename to packages/test-kit/src/validation-values.ts index 6d37e1fc53..7ce99fcb4a 100644 --- a/scripts/fuzz/validation-values.ts +++ b/packages/test-kit/src/validation-values.ts @@ -19,3 +19,15 @@ export const SAFE_VALUES = [ /** The accept expectation, shared so a valid case reads the same in both generators. */ export const ACCEPT = { outcome: 'accept' } as const; + +export type ValidationExpectation = { outcome: 'accept' } | { outcome: 'reject'; code: string }; + +export type ValidationCase = { + payload: string[] | string; + mutation: string; + expect: ValidationExpectation; +}; + +export function encodeValidationCase(validationCase: ValidationCase): string { + return JSON.stringify(validationCase); +} diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts index d604e2e0fa..746ef0b7af 100644 --- a/scripts/fuzz/envelope.ts +++ b/scripts/fuzz/envelope.ts @@ -90,8 +90,9 @@ export const CASE_GENERATION_INPUTS = [ 'invariant.ts', 'validation-arbitraries.ts', 'validation-arbitraries-cli.ts', - 'validation-arbitraries-maestro.ts', - 'validation-values.ts', + '../../packages/test-kit/src/maestro-validation-arbitraries.ts', + '../../packages/test-kit/src/validation-values.ts', + '../../packages/test-kit/src/index.ts', 'validation-case.ts', '../../src/commands/replay/replay-script.fixtures.ts', ] as const; diff --git a/scripts/fuzz/validation-arbitraries-cli.ts b/scripts/fuzz/validation-arbitraries-cli.ts index 05d4a6a57c..c50daf01bd 100644 --- a/scripts/fuzz/validation-arbitraries-cli.ts +++ b/scripts/fuzz/validation-arbitraries-cli.ts @@ -13,7 +13,7 @@ import { getFlagDefinitions } from '@agent-device/command-registry/flag-registry import type { FlagDefinition } from '@agent-device/command-registry/flag-types'; import { getCliCommandSchema, isFlagSupportedForCommand } from '@agent-device/command-surface'; import { encodeValidationCase, type ValidationCase } from './validation-case.ts'; -import { ACCEPT, SAFE_VALUES } from './validation-values.ts'; +import { ACCEPT, SAFE_VALUES } from '@agent-device/test-kit'; type CliCommandSurface = { name: string; diff --git a/scripts/fuzz/validation-arbitraries-maestro.test.ts b/scripts/fuzz/validation-arbitraries-maestro.test.ts index f52ec51160..1fa0a134e0 100644 --- a/scripts/fuzz/validation-arbitraries-maestro.test.ts +++ b/scripts/fuzz/validation-arbitraries-maestro.test.ts @@ -8,11 +8,10 @@ import fc from 'fast-check'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { SUPPORTED_MAESTRO_COMMAND_NAMES } from '../../packages/maestro/src/internal/program-ir-command-parser.ts'; import { getFuzzTarget } from './registry.ts'; import { decodeValidationCase } from './validation-case.ts'; import { describeGeneratorContract } from './validation-generator-contract.ts'; -import { MAESTRO_MUTATION_NAMES, maestroValidationArb } from './validation-arbitraries-maestro.ts'; +import { MAESTRO_MUTATION_NAMES, maestroValidationArb } from '@agent-device/test-kit'; const SAMPLE_SIZE = 3_000; const SEED = 1; @@ -63,30 +62,3 @@ describe('planted Maestro violations are refused by command-shape validation', ( ); }); }); - -describe('shape coverage against the converter', () => { - // The valid Maestro shapes are rendered by hand here, so the one thing that cannot be left to - // drift is which commands they cover: a command the converter accepts but this generator never - // emits is a hole the nightly cannot see. Waivers are explicit and must say why. - const NOT_GENERATED = new Set([ - // Both take a nested command list whose bodies the repeat/runFlow cases already cover. - 'retry', - // Needs a JS file on disk to resolve, which a generated case has no way to provide. - 'runScript', - ]); - - it('emits every command the converter supports, or waives it explicitly', () => { - const emitted = new Set( - sample.flatMap((entry) => { - const { payload } = decodeValidationCase(entry)!; - return typeof payload === 'string' ? payload.split('\n') : []; - }), - ); - const covered = (command: string) => - [...emitted].some((line) => line.trim().startsWith(`- ${command}`)); - const missing = SUPPORTED_MAESTRO_COMMAND_NAMES.filter( - (command) => !NOT_GENERATED.has(command) && !covered(command), - ); - expect(missing).toEqual([]); - }); -}); diff --git a/scripts/fuzz/validation-arbitraries.ts b/scripts/fuzz/validation-arbitraries.ts index 81799b7a08..a532834ad8 100644 --- a/scripts/fuzz/validation-arbitraries.ts +++ b/scripts/fuzz/validation-arbitraries.ts @@ -7,7 +7,7 @@ import type fc from 'fast-check'; import type { FuzzTargetName } from './target-types.ts'; import { cliValidationArb } from './validation-arbitraries-cli.ts'; -import { maestroValidationArb } from './validation-arbitraries-maestro.ts'; +import { maestroValidationArb } from '@agent-device/test-kit'; /** The generator for a validation target, or `undefined` for the classic targets. */ export function validationArbitraryFor(target: FuzzTargetName): fc.Arbitrary | undefined { diff --git a/scripts/fuzz/validation-case.ts b/scripts/fuzz/validation-case.ts index 8f59421df9..80c94a7ffa 100644 --- a/scripts/fuzz/validation-case.ts +++ b/scripts/fuzz/validation-case.ts @@ -1,3 +1,9 @@ +import { + encodeValidationCase, + type ValidationCase, + type ValidationExpectation, +} from '@agent-device/test-kit'; +export { encodeValidationCase, type ValidationCase } from '@agent-device/test-kit'; // Expectation-carrying cases for the validation fuzz targets (#1781 B2). // // The classic targets judge only rejections ("fails well"), which cannot see the bug class @@ -11,20 +17,6 @@ import { AppError, normalizeError } from '@agent-device/kernel/errors'; import { describeThrown } from './invariant.ts'; import type { FuzzFailure, FuzzTargetName } from './target-types.ts'; -export type ValidationExpectation = { outcome: 'accept' } | { outcome: 'reject'; code: string }; - -export type ValidationCase = { - /** CLI argv vector or Maestro flow source, depending on the target. */ - payload: string[] | string; - /** Which generator rule produced the case — names the finding and the calibration row. */ - mutation: string; - expect: ValidationExpectation; -}; - -export function encodeValidationCase(validationCase: ValidationCase): string { - return JSON.stringify(validationCase); -} - /** A pinned case the parser must accept. */ export function acceptCase(payload: ValidationCase['payload']): string { return encodeValidationCase({ payload, mutation: 'valid', expect: { outcome: 'accept' } }); From 74c5f78cadc51df189fc0de06e7e8bd5af178e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 23:19:15 +0100 Subject: [PATCH 2/9] fix(host-kit): bound captured command output --- packages/host-kit/src/internal/exec-trace.ts | 1 + packages/host-kit/src/internal/exec.test.ts | 23 +++- packages/host-kit/src/internal/exec.ts | 115 ++++++++++++------- 3 files changed, 97 insertions(+), 42 deletions(-) diff --git a/packages/host-kit/src/internal/exec-trace.ts b/packages/host-kit/src/internal/exec-trace.ts index 9b1f516476..8960b0b9cb 100644 --- a/packages/host-kit/src/internal/exec-trace.ts +++ b/packages/host-kit/src/internal/exec-trace.ts @@ -16,6 +16,7 @@ export type ExecSettlementReason = | 'success' | 'exit_failure' | 'timeout' + | 'output_limit' | 'abort' | 'spawn_error' | 'stdin_error'; diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index 6009a3f6a9..7f5d19e7d7 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -1,4 +1,4 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; @@ -500,3 +500,24 @@ test.skipIf(process.platform === 'win32')( assert.equal('signal' in exited, false); }, ); + +test('async captured output enforces a per-stream byte limit', async () => { + for (const stream of ['stdout', 'stderr'] as const) { + const emit = `process.${stream}.write('x'.repeat(Number(process.argv[1])))`; + const within = await runCmd(process.execPath, ['-e', emit, '128'], { maxBuffer: 128 }); + expect(within[stream]).toHaveLength(128); + await expect( + runCmd(process.execPath, ['-e', emit, '129'], { maxBuffer: 128 }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'command_output_limit', stream, maxBuffer: 128 }, + }); + } + const separate = await runCmd( + process.execPath, + ['-e', "process.stdout.write('x'.repeat(128)); process.stderr.write('y'.repeat(128));"], + { maxBuffer: 128 }, + ); + expect(separate.stdout).toHaveLength(128); + expect(separate.stderr).toHaveLength(128); +}); diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index e9611933a6..134e3ebd3e 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -27,7 +27,7 @@ export type ExecOptions = { timeoutMs?: number; detached?: boolean; signal?: AbortSignal; - /** Max stdout/stderr bytes for synchronous runs (default Node ~1MB). */ + /** Max captured bytes per output stream. Async runs are unlimited when omitted. */ maxBuffer?: number; /** * How a deadline or a cancelled request stops the child. Without this it is SIGKILL at @@ -227,6 +227,8 @@ function runSpawnedCommand( const stdoutChunks: Buffer[] | undefined = options.binaryStdout && options.captureOutput !== false ? [] : undefined; let stderr = ''; + const outputBytes = { stdout: 0, stderr: 0 }; + let exceededStream: 'stdout' | 'stderr' | undefined; let didTimeout = false; const timeoutMs = normalizeTimeoutMs(options.timeoutMs); let timeoutHandle: NodeJS.Timeout | null = null; @@ -244,15 +246,27 @@ function runSpawnedCommand( if (finish(reason)) reject(error); } function settle(code: number | null): void { - const finalReason = abort.didAbort - ? 'abort' - : didTimeout - ? 'timeout' - : (code ?? 1) !== 0 && !options.allowFailure - ? 'exit_failure' - : 'success'; + const finalReason = commandSettlementReason({ + abort, + exceededStream, + didTimeout, + code, + allowFailure: options.allowFailure, + }); if (!finish(finalReason)) return; const finalExitCode = code ?? 1; + if (!abort.didAbort && exceededStream) { + reject( + new AppError('COMMAND_FAILED', `${executable} exceeded its output limit`, { + cmd, + args, + reason: 'command_output_limit', + stream: exceededStream, + maxBuffer: options.maxBuffer, + }), + ); + return; + } if (!abort.didAbort && didTimeout && timeoutMs) { reject(createTimeoutError(executable, cmd, args, timeoutMs, finalExitCode, stdout, stderr)); return; @@ -283,14 +297,7 @@ function runSpawnedCommand( // A deadline that fires after the child exited on its own still has to settle: the // group kill that would have ended the pipe holder can no longer run through a child // Node already reaped. - const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options), - settle, - }); - const abort = watchCommandAbort(options, () => { - execTrace.emit('abort'); - settlement.requestKill(); - }); + const { settlement, abort } = bindCommandLifecycle(child, options, execTrace, settle); timeoutHandle = timeoutMs ? setTimeout(() => { execTrace.emit('deadline'); @@ -309,7 +316,18 @@ function runSpawnedCommand( killProcessTree(child, options); }); + function acceptOutput(stream: 'stdout' | 'stderr', chunk: string | Buffer): boolean { + if (exceededStream) return false; + if (options.captureOutput === false || options.maxBuffer === undefined) return true; + outputBytes[stream] += Buffer.byteLength(chunk); + if (outputBytes[stream] <= options.maxBuffer) return true; + exceededStream = stream; + settlement.requestKill(); + return false; + } + child.stdout.on('data', (chunk) => { + if (!acceptOutput('stdout', chunk)) return; if (options.binaryStdout) { stdoutChunks?.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); return; @@ -320,6 +338,7 @@ function runSpawnedCommand( }); child.stderr.on('data', (chunk) => { + if (!acceptOutput('stderr', chunk)) return; const text = String(chunk); if (options.captureOutput !== false) stderr += text; options.onStderrChunk?.(text); @@ -329,15 +348,6 @@ function runSpawnedCommand( execTrace.emit('error'); fail(spawnRejectionError(abort, executable, cmd, args, err), 'spawn_error'); }); - - child.once('exit', (code, signal) => { - execTrace.emit('exit', { exitCode: code, signal }); - settlement.recordExit(code); - }); - child.once('close', (code, signal) => { - execTrace.emit('close', { exitCode: code, signal }); - settle(code); - }); }); } @@ -572,26 +582,11 @@ export function runCmdBackground( } resolve({ stdout, stderr, exitCode: finalExitCode }); } - const settlement = createCommandKillSettlement({ - killProcessTree: () => killProcessTree(child, options), - settle, - }); - const abort = watchCommandAbort(options, () => { - execTrace.emit('abort'); - settlement.requestKill(); - }); + const { abort } = bindCommandLifecycle(child, options, execTrace, settle); child.on('error', (err) => { execTrace.emit('error'); if (finish('error')) reject(spawnRejectionError(abort, executable, cmd, args, err)); }); - child.once('exit', (code, signal) => { - execTrace.emit('exit', { exitCode: code, signal }); - settlement.recordExit(code); - }); - child.once('close', (code, signal) => { - execTrace.emit('close', { exitCode: code, signal }); - settle(code); - }); }); return { child, wait }; @@ -866,6 +861,44 @@ function normalizeTimeoutMs(value: number | undefined): number | undefined { * A killed command settles when its direct child exits: descendants can keep inherited pipes * open indefinitely. Both kill/exit orderings must settle once without waiting for `close`. */ +function commandSettlementReason(input: { + abort: CommandAbort; + exceededStream: 'stdout' | 'stderr' | undefined; + didTimeout: boolean; + code: number | null; + allowFailure: boolean | undefined; +}): ExecSettlementReason { + if (input.abort.didAbort) return 'abort'; + if (input.exceededStream) return 'output_limit'; + if (input.didTimeout) return 'timeout'; + return (input.code ?? 1) !== 0 && !input.allowFailure ? 'exit_failure' : 'success'; +} + +function bindCommandLifecycle( + child: ReturnType, + options: Pick, + trace: ReturnType, + settle: (code: number | null) => void, +) { + const settlement = createCommandKillSettlement({ + killProcessTree: () => killProcessTree(child, options), + settle, + }); + const abort = watchCommandAbort(options, () => { + trace.emit('abort'); + settlement.requestKill(); + }); + child.once('exit', (code, signal) => { + trace.emit('exit', { exitCode: code, signal }); + settlement.recordExit(code); + }); + child.once('close', (code, signal) => { + trace.emit('close', { exitCode: code, signal }); + settle(code); + }); + return { settlement, abort }; +} + type CommandKillSettlement = { /** Signals the command's process tree, then settles the command if its child is gone. */ readonly requestKill: () => void; From efee144bd0002ce9c00aeb7bf3e8a6eed0f8b56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 23:14:15 +0100 Subject: [PATCH 3/9] fix(host-kit): count raw output bytes before decoding --- packages/host-kit/src/internal/exec.test.ts | 39 +++++++++++++++++++++ packages/host-kit/src/internal/exec.ts | 33 +++++++++++------ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index 7f5d19e7d7..ad3a2f47ec 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -521,3 +521,42 @@ test('async captured output enforces a per-stream byte limit', async () => { expect(separate.stdout).toHaveLength(128); expect(separate.stderr).toHaveLength(128); }); + + +test('async output limits count raw bytes before decoding text', async () => { + for (const stream of ['stdout', 'stderr'] as const) { + for (const bytes of [[0xff], [0xe2, 0x82]]) { + const result = await runCmd( + process.execPath, + ['-e', 'process.' + stream + '.write(Buffer.from(' + JSON.stringify(bytes) + '))'], + { maxBuffer: bytes.length }, + ); + expect(result[stream]).toBe('\uFFFD'); + } + await expect( + runCmd( + process.execPath, + ['-e', 'process.' + stream + '.write(Buffer.from([0xff, 0xff]))'], + { maxBuffer: 1 }, + ), + ).rejects.toMatchObject({ details: { reason: 'command_output_limit', stream, maxBuffer: 1 } }); + } + const binary = await runCmd( + process.execPath, + ['-e', 'process.stdout.write(Buffer.from([0xff]))'], + { binaryStdout: true, maxBuffer: 1 }, + ); + expect(binary.stdoutBuffer).toEqual(Buffer.from([0xff])); +}); + +test('bounded text output preserves multibyte characters across pipe chunks', async () => { + const expected = '\u20AC'.repeat(50_000); + for (const stream of ['stdout', 'stderr'] as const) { + const result = await runCmd( + process.execPath, + ['-e', 'process.' + stream + '.write(String.fromCharCode(0x20ac).repeat(50000))'], + { maxBuffer: Buffer.byteLength(expected) }, + ); + expect(result[stream]).toBe(expected); + } +}); diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 134e3ebd3e..0c0c1bb8da 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { spawn, spawnSync, type ChildProcess, type StdioOptions } from 'node:child_process'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; +import { StringDecoder } from 'node:string_decoder'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { createExecTraceContext, type ExecSettlementReason } from './exec-trace.ts'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; @@ -306,8 +307,8 @@ function runSpawnedCommand( }, timeoutMs) : null; - if (!options.binaryStdout) child.stdout.setEncoding('utf8'); - child.stderr.setEncoding('utf8'); + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); void writeChildStdin(child, options.stdin).catch((error: unknown) => { if (abort.didAbort || didTimeout) return; @@ -326,22 +327,34 @@ function runSpawnedCommand( return false; } - child.stdout.on('data', (chunk) => { + function appendText(stream: 'stdout' | 'stderr', text: string): void { + if (!text) return; + if (options.captureOutput !== false) { + if (stream === 'stdout') stdout += text; + else stderr += text; + } + if (stream === 'stdout') options.onStdoutChunk?.(text); + else options.onStderrChunk?.(text); + } + + child.stdout.on('data', (chunk: Buffer) => { if (!acceptOutput('stdout', chunk)) return; if (options.binaryStdout) { stdoutChunks?.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); return; } - const text = String(chunk); - if (options.captureOutput !== false) stdout += text; - options.onStdoutChunk?.(text); + appendText('stdout', stdoutDecoder.write(chunk)); }); - child.stderr.on('data', (chunk) => { + child.stderr.on('data', (chunk: Buffer) => { if (!acceptOutput('stderr', chunk)) return; - const text = String(chunk); - if (options.captureOutput !== false) stderr += text; - options.onStderrChunk?.(text); + appendText('stderr', stderrDecoder.write(chunk)); + }); + child.stdout.on('end', () => { + if (!options.binaryStdout && !exceededStream) appendText('stdout', stdoutDecoder.end()); + }); + child.stderr.on('end', () => { + if (!exceededStream) appendText('stderr', stderrDecoder.end()); }); child.on('error', (err) => { From 6bf6ee5c3d372a6eb3f38a8a6cbc9b379b95c7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 23:21:59 +0100 Subject: [PATCH 4/9] refactor(host-kit): isolate command settlement classification --- .../host-kit/src/internal/exec-settlement.ts | 14 ++++++++++++++ packages/host-kit/src/internal/exec.test.ts | 9 +++------ packages/host-kit/src/internal/exec.ts | 16 ++-------------- 3 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 packages/host-kit/src/internal/exec-settlement.ts diff --git a/packages/host-kit/src/internal/exec-settlement.ts b/packages/host-kit/src/internal/exec-settlement.ts new file mode 100644 index 0000000000..0a78f4fbb2 --- /dev/null +++ b/packages/host-kit/src/internal/exec-settlement.ts @@ -0,0 +1,14 @@ +import type { ExecSettlementReason } from './exec-trace.ts'; + +export function commandSettlementReason(input: { + didAbort: boolean; + exceededStream: 'stdout' | 'stderr' | undefined; + didTimeout: boolean; + code: number | null; + allowFailure: boolean | undefined; +}): ExecSettlementReason { + if (input.didAbort) return 'abort'; + if (input.exceededStream) return 'output_limit'; + if (input.didTimeout) return 'timeout'; + return (input.code ?? 1) !== 0 && !input.allowFailure ? 'exit_failure' : 'success'; +} diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index ad3a2f47ec..fc3612c1ee 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -522,7 +522,6 @@ test('async captured output enforces a per-stream byte limit', async () => { expect(separate.stderr).toHaveLength(128); }); - test('async output limits count raw bytes before decoding text', async () => { for (const stream of ['stdout', 'stderr'] as const) { for (const bytes of [[0xff], [0xe2, 0x82]]) { @@ -534,11 +533,9 @@ test('async output limits count raw bytes before decoding text', async () => { expect(result[stream]).toBe('\uFFFD'); } await expect( - runCmd( - process.execPath, - ['-e', 'process.' + stream + '.write(Buffer.from([0xff, 0xff]))'], - { maxBuffer: 1 }, - ), + runCmd(process.execPath, ['-e', 'process.' + stream + '.write(Buffer.from([0xff, 0xff]))'], { + maxBuffer: 1, + }), ).rejects.toMatchObject({ details: { reason: 'command_output_limit', stream, maxBuffer: 1 } }); } const binary = await runCmd( diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 0c0c1bb8da..1341b06656 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -1,3 +1,4 @@ +import { commandSettlementReason } from './exec-settlement.ts'; import { AsyncLocalStorage } from 'node:async_hooks'; import { constants } from 'node:fs'; import { access, stat } from 'node:fs/promises'; @@ -248,7 +249,7 @@ function runSpawnedCommand( } function settle(code: number | null): void { const finalReason = commandSettlementReason({ - abort, + didAbort: abort.didAbort, exceededStream, didTimeout, code, @@ -364,7 +365,6 @@ function runSpawnedCommand( }); } -/** Returns the result with `signal` added when a signal terminated the process. */ function execResult(ended: { signalCode: string | null }, result: ExecResult): ExecResult { return ended.signalCode ? { ...result, signal: ended.signalCode } : result; } @@ -874,18 +874,6 @@ function normalizeTimeoutMs(value: number | undefined): number | undefined { * A killed command settles when its direct child exits: descendants can keep inherited pipes * open indefinitely. Both kill/exit orderings must settle once without waiting for `close`. */ -function commandSettlementReason(input: { - abort: CommandAbort; - exceededStream: 'stdout' | 'stderr' | undefined; - didTimeout: boolean; - code: number | null; - allowFailure: boolean | undefined; -}): ExecSettlementReason { - if (input.abort.didAbort) return 'abort'; - if (input.exceededStream) return 'output_limit'; - if (input.didTimeout) return 'timeout'; - return (input.code ?? 1) !== 0 && !input.allowFailure ? 'exit_failure' : 'success'; -} function bindCommandLifecycle( child: ReturnType, From 0d6b9f6d13f9533c33fdbdd94f565a20393f41bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 23:34:03 +0100 Subject: [PATCH 5/9] fix(host-kit): preserve timeout and eager closure --- .../host-kit/src/internal/exec-settlement.ts | 14 --------- packages/host-kit/src/internal/exec-trace.ts | 20 ++++++++++++ packages/host-kit/src/internal/exec.test.ts | 31 +++++++++++++++++++ packages/host-kit/src/internal/exec.ts | 18 +++++------ 4 files changed, 59 insertions(+), 24 deletions(-) delete mode 100644 packages/host-kit/src/internal/exec-settlement.ts diff --git a/packages/host-kit/src/internal/exec-settlement.ts b/packages/host-kit/src/internal/exec-settlement.ts deleted file mode 100644 index 0a78f4fbb2..0000000000 --- a/packages/host-kit/src/internal/exec-settlement.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { ExecSettlementReason } from './exec-trace.ts'; - -export function commandSettlementReason(input: { - didAbort: boolean; - exceededStream: 'stdout' | 'stderr' | undefined; - didTimeout: boolean; - code: number | null; - allowFailure: boolean | undefined; -}): ExecSettlementReason { - if (input.didAbort) return 'abort'; - if (input.exceededStream) return 'output_limit'; - if (input.didTimeout) return 'timeout'; - return (input.code ?? 1) !== 0 && !input.allowFailure ? 'exit_failure' : 'success'; -} diff --git a/packages/host-kit/src/internal/exec-trace.ts b/packages/host-kit/src/internal/exec-trace.ts index 8960b0b9cb..2f740638ce 100644 --- a/packages/host-kit/src/internal/exec-trace.ts +++ b/packages/host-kit/src/internal/exec-trace.ts @@ -69,3 +69,23 @@ export function createExecTraceContext( emit('spawn_start'); return { emit }; } + +export function commandSettlementReason(input: { + didAbort: boolean; + exceededStream: 'stdout' | 'stderr' | undefined; + didTimeout: boolean; + code: number | null; + allowFailure: boolean | undefined; +}): ExecSettlementReason { + if (input.didAbort) return 'abort'; + if (input.didTimeout) return 'timeout'; + if (input.exceededStream) return 'output_limit'; + return (input.code ?? 1) !== 0 && !input.allowFailure ? 'exit_failure' : 'success'; +} + +export function normalizeTimeoutMs(value: number | undefined): number | undefined { + if (!Number.isFinite(value)) return undefined; + const timeout = Math.floor(value as number); + if (timeout <= 0) return undefined; + return timeout; +} diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index fc3612c1ee..4c8b1d1e9c 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -557,3 +557,34 @@ test('bounded text output preserves multibyte characters across pipe chunks', as expect(result[stream]).toBe(expected); } }); + +test.skipIf(process.platform === 'win32')( + 'graceful shutdown output does not replace an earlier timeout', + async () => { + let stdout = ''; + const error = await runCmdStreaming( + process.execPath, + [ + '-e', + [ + "process.on('SIGTERM', () => { process.stderr.write('x'.repeat(129)); clearInterval(keepAlive); });", + 'const keepAlive = setInterval(() => {}, 100);', + "process.stdout.write('ready');", + ].join('\n'), + ], + { + timeoutMs: 1_000, + maxBuffer: 128, + kill: { signal: 'SIGTERM', graceMs: 1_000 }, + onStdoutChunk: (chunk) => { + stdout += chunk; + }, + }, + ).then( + () => undefined, + (error: unknown) => error, + ); + assert.equal(stdout, 'ready'); + assert.ok(isCommandTimeoutError(error)); + }, +); diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 1341b06656..6cfe7e81a4 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -1,4 +1,3 @@ -import { commandSettlementReason } from './exec-settlement.ts'; import { AsyncLocalStorage } from 'node:async_hooks'; import { constants } from 'node:fs'; import { access, stat } from 'node:fs/promises'; @@ -8,7 +7,12 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { StringDecoder } from 'node:string_decoder'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; -import { createExecTraceContext, type ExecSettlementReason } from './exec-trace.ts'; +import { + commandSettlementReason, + createExecTraceContext, + normalizeTimeoutMs, + type ExecSettlementReason, +} from './exec-trace.ts'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; export type ExecResult = { @@ -257,7 +261,7 @@ function runSpawnedCommand( }); if (!finish(finalReason)) return; const finalExitCode = code ?? 1; - if (!abort.didAbort && exceededStream) { + if (!abort.didAbort && exceededStream && !didTimeout) { reject( new AppError('COMMAND_FAILED', `${executable} exceeded its output limit`, { cmd, @@ -302,6 +306,7 @@ function runSpawnedCommand( const { settlement, abort } = bindCommandLifecycle(child, options, execTrace, settle); timeoutHandle = timeoutMs ? setTimeout(() => { + if (exceededStream) return; execTrace.emit('deadline'); didTimeout = true; settlement.requestKill(); @@ -863,13 +868,6 @@ async function isFilePath(filePath: string): Promise { } } -function normalizeTimeoutMs(value: number | undefined): number | undefined { - if (!Number.isFinite(value)) return undefined; - const timeout = Math.floor(value as number); - if (timeout <= 0) return undefined; - return timeout; -} - /** * A killed command settles when its direct child exits: descendants can keep inherited pipes * open indefinitely. Both kill/exit orderings must settle once without waiting for `close`. From c78fe581c4df431112c10a3fdfa315950048b728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 08:15:35 +0100 Subject: [PATCH 6/9] fix(test-kit): cover complete Maestro command tokens --- packages/host-kit/src/internal/exec.ts | 4 +-- ...am-ir-command-parser-fuzz-coverage.test.ts | 26 ++++++++++--------- .../src/maestro-validation-arbitraries.ts | 7 +++++ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 6cfe7e81a4..38bee3533e 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -33,7 +33,7 @@ export type ExecOptions = { timeoutMs?: number; detached?: boolean; signal?: AbortSignal; - /** Max captured bytes per output stream. Async runs are unlimited when omitted. */ + /** Max captured bytes per stream for runCmd and runCmdSync. runCmd is unlimited when omitted. */ maxBuffer?: number; /** * How a deadline or a cancelled request stops the child. Without this it is SIGKILL at @@ -82,7 +82,7 @@ export type ExecDetachedProcess = { * `signal`; a caller that kills the child directly still waits for the streams to * drain, exactly as before. */ -export type ExecBackgroundOptions = Omit & { +export type ExecBackgroundOptions = Omit & { /** * Capture stdout/stderr into the wait result when the child has piped stdio. * Set false when the caller owns, ignores, or forwards the streams. diff --git a/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts index 4ed0ec8395..394e8cee54 100644 --- a/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts +++ b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts @@ -5,18 +5,20 @@ import { SUPPORTED_MAESTRO_COMMAND_NAMES } from './program-ir-command-parser.ts' const sample = fc.sample(maestroValidationArb, { numRuns: 3_000, seed: 1 }); +function coversCommand(line: string, command: string): boolean { + const trimmed = line.trim(); + return trimmed === `- ${command}` || trimmed.startsWith(`- ${command}:`); +} + describe('shape coverage against the converter', () => { - // The valid Maestro shapes are rendered by hand here, so the one thing that cannot be left to - // drift is which commands they cover: a command the converter accepts but this generator never - // emits is a hole the nightly cannot see. Waivers are explicit and must say why. - const NOT_GENERATED = new Set([ - // Both take a nested command list whose bodies the repeat/runFlow cases already cover. - 'retry', - // Needs a JS file on disk to resolve, which a generated case has no way to provide. - 'runScript', - ]); + it('does not count invalid command-name prefixes as coverage', () => { + expect(coversCommand('- tapOnPoint: 10,20', 'tapOn')).toBe(false); + expect(coversCommand('- inputTextt: Login', 'inputText')).toBe(false); + expect(coversCommand('- tapOn: Login', 'tapOn')).toBe(true); + expect(coversCommand(' - back', 'back')).toBe(true); + }); - it('emits every command the converter supports, or waives it explicitly', () => { + it('emits every command the converter supports', () => { const emitted = new Set( sample.flatMap((entry) => { const { payload } = JSON.parse(entry) as { payload: string | string[] }; @@ -24,9 +26,9 @@ describe('shape coverage against the converter', () => { }), ); const covered = (command: string) => - [...emitted].some((line) => line.trim().startsWith(`- ${command}`)); + [...emitted].some((line) => coversCommand(line, command)); const missing = SUPPORTED_MAESTRO_COMMAND_NAMES.filter( - (command) => !NOT_GENERATED.has(command) && !covered(command), + (command) => !covered(command), ); expect(missing).toEqual([]); }); diff --git a/packages/test-kit/src/maestro-validation-arbitraries.ts b/packages/test-kit/src/maestro-validation-arbitraries.ts index 57b593c873..0e924dc013 100644 --- a/packages/test-kit/src/maestro-validation-arbitraries.ts +++ b/packages/test-kit/src/maestro-validation-arbitraries.ts @@ -48,6 +48,8 @@ function validMaestroCommand(pick: number, salt: number): string[] { () => ['- scrollUntilVisible:', ' element:', ` text: ${text}`], () => ['- repeat:', ' times: 2', ' commands:', ' - back'], () => ['- runFlow: other.yaml'], + () => ['- retry:', ' maxRetries: 2', ' commands:', ' - back'], + () => ['- runScript: checks.js'], () => ['- setPermissions:', ' permissions:', ' camera: allow'], () => ['- setPermissions:', ' permissions:', ' all: deny'], () => ['- launchApp:', ' appId: com.example.app', ' permissions:', ' camera: allow'], @@ -91,6 +93,11 @@ const MAESTRO_MUTATIONS: readonly MaestroMutation[] = [ lines: (salt) => salt % 2 === 0 ? ['- inputText:'] : ['- extendedWaitUntil:', ' timeout: 500'], }, + { + name: 'bad-retry-max-retries', + code: 'INVALID_ARGS', + lines: () => ['- retry:', ' maxRetries: invalid', ' commands:', ' - back'], + }, { name: 'bad-press-key', code: 'INVALID_ARGS', lines: () => ['- pressKey: sleep'] }, { name: 'scroll-options', code: 'INVALID_ARGS', lines: () => ['- scroll:', ' direction: UP'] }, { From f5891f9d7feef6a08685124cebaea4cf035fd9b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 08:18:12 +0100 Subject: [PATCH 7/9] style(test): format Maestro coverage assertion --- .../program-ir-command-parser-fuzz-coverage.test.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts index 394e8cee54..08df4493c9 100644 --- a/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts +++ b/packages/maestro/src/internal/program-ir-command-parser-fuzz-coverage.test.ts @@ -25,11 +25,8 @@ describe('shape coverage against the converter', () => { return typeof payload === 'string' ? payload.split('\n') : []; }), ); - const covered = (command: string) => - [...emitted].some((line) => coversCommand(line, command)); - const missing = SUPPORTED_MAESTRO_COMMAND_NAMES.filter( - (command) => !covered(command), - ); + const covered = (command: string) => [...emitted].some((line) => coversCommand(line, command)); + const missing = SUPPORTED_MAESTRO_COMMAND_NAMES.filter((command) => !covered(command)); expect(missing).toEqual([]); }); }); From 9a83e349a9b1caa83668e500abfabd6679173853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 09:32:51 +0100 Subject: [PATCH 8/9] fix(host-kit): retain bounded output on capture limit errors --- packages/host-kit/src/internal/exec.test.ts | 19 +++++++++++++++++++ packages/host-kit/src/internal/exec.ts | 2 ++ 2 files changed, 21 insertions(+) diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index 4c8b1d1e9c..8dacd61aab 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -522,6 +522,25 @@ test('async captured output enforces a per-stream byte limit', async () => { expect(separate.stderr).toHaveLength(128); }); +test('an output-limit failure retains bounded output with allowFailure enabled', async () => { + for (const stream of ['stdout', 'stderr'] as const) { + await expect( + runCmd(process.execPath, ['-e', `process.${stream}.write('x'.repeat(129))`], { + maxBuffer: 128, + allowFailure: true, + }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason: 'command_output_limit', + stream, + maxBuffer: 128, + [stream]: 'x'.repeat(128), + }, + }); + } +}); + test('async output limits count raw bytes before decoding text', async () => { for (const stream of ['stdout', 'stderr'] as const) { for (const bytes of [[0xff], [0xe2, 0x82]]) { diff --git a/packages/host-kit/src/internal/exec.ts b/packages/host-kit/src/internal/exec.ts index 38bee3533e..3c90680f70 100644 --- a/packages/host-kit/src/internal/exec.ts +++ b/packages/host-kit/src/internal/exec.ts @@ -269,6 +269,8 @@ function runSpawnedCommand( reason: 'command_output_limit', stream: exceededStream, maxBuffer: options.maxBuffer, + stdout, + stderr, }), ); return; From 0399650fa8c516d7e5a7c1e4389e5f05d52e11e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 09:37:36 +0100 Subject: [PATCH 9/9] test: coordinate bounded output regression --- packages/host-kit/src/internal/exec.test.ts | 56 ++++++++++++++------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/packages/host-kit/src/internal/exec.test.ts b/packages/host-kit/src/internal/exec.test.ts index 8dacd61aab..904b1ad8ce 100644 --- a/packages/host-kit/src/internal/exec.test.ts +++ b/packages/host-kit/src/internal/exec.test.ts @@ -522,24 +522,44 @@ test('async captured output enforces a per-stream byte limit', async () => { expect(separate.stderr).toHaveLength(128); }); -test('an output-limit failure retains bounded output with allowFailure enabled', async () => { - for (const stream of ['stdout', 'stderr'] as const) { - await expect( - runCmd(process.execPath, ['-e', `process.${stream}.write('x'.repeat(129))`], { - maxBuffer: 128, - allowFailure: true, - }), - ).rejects.toMatchObject({ - code: 'COMMAND_FAILED', - details: { - reason: 'command_output_limit', - stream, - maxBuffer: 128, - [stream]: 'x'.repeat(128), - }, - }); - } -}); +test.skipIf(process.platform === 'win32')( + 'an output-limit failure retains bounded output with allowFailure enabled', + async () => { + for (const stream of ['stdout', 'stderr'] as const) { + let child: ReturnType | undefined; + let received = ''; + const onChunk = (chunk: string) => { + received += chunk; + if (received.length === 128) child?.kill('SIGUSR1'); + }; + await expect( + runCmdStreaming( + process.execPath, + [ + '-e', + `process.on('SIGUSR1', () => process.${stream}.write('x')); setInterval(() => {}, 1000); process.${stream}.write('x'.repeat(128));`, + ], + { + maxBuffer: 128, + allowFailure: true, + onSpawn: (spawned) => { + child = spawned; + }, + ...(stream === 'stdout' ? { onStdoutChunk: onChunk } : { onStderrChunk: onChunk }), + }, + ), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason: 'command_output_limit', + stream, + maxBuffer: 128, + [stream]: 'x'.repeat(128), + }, + }); + } + }, +); test('async output limits count raw bytes before decoding text', async () => { for (const stream of ['stdout', 'stderr'] as const) {