From 2d85dcd8e24234310357eedd0ca43dccdf9ce2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 20:53:08 +0100 Subject: [PATCH 1/4] fix(proxy): bound transfer phases and join owned cleanup --- .../http/http-server-proxy-rollback.test.ts | 163 ++++ .../src/internal/transport-phases.test.ts | 32 + packages/host-kit/src/internal/transport.ts | 12 +- packages/host-kit/src/transport.ts | 3 + packages/proxy/README.md | 54 +- .../proxy/src/daemon-proxy-transfer.test.ts | 408 +++++++++ packages/proxy/src/daemon-proxy.ts | 57 +- .../src/transfer-transport-lifetime.test.ts | 803 ++++++++++++++++++ packages/proxy/src/transfer-transport.test.ts | 409 +++++++++ packages/proxy/src/transfer-transport.ts | 481 +++++++++++ test/wire-compat/ledger.json | 7 +- 11 files changed, 2409 insertions(+), 20 deletions(-) create mode 100644 packages/daemon/src/server/http/http-server-proxy-rollback.test.ts create mode 100644 packages/proxy/src/daemon-proxy-transfer.test.ts create mode 100644 packages/proxy/src/transfer-transport-lifetime.test.ts create mode 100644 packages/proxy/src/transfer-transport.test.ts create mode 100644 packages/proxy/src/transfer-transport.ts diff --git a/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts new file mode 100644 index 0000000000..e06e536d6f --- /dev/null +++ b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts @@ -0,0 +1,163 @@ +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { createDaemonProxyServer } from '@agent-device/proxy'; +import { + closeLoopbackServer, + createTestRuntimeStateHandle, + listenOnLoopback, +} from '@agent-device/test-kit'; +import { + cleanupUploadedArtifact, + prepareUploadedArtifact, +} from '../../resources/artifact-tracking.ts'; +import { createDaemonHttpServer } from './http-server.ts'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +test.each(['disconnect and resume', 'uninterrupted'] as const)( + 'an actual proxy-to-daemon upload handles %s and verifies finalized bytes', + async (mode) => { + const runtime = createTestRuntimeStateHandle(); + try { + await runtime.run(async () => { + const daemon = await createDaemonHttpServer({ + token: 'daemon-secret', + handleRequest: async () => ({ ok: true, data: {} }), + env: process.env, + cwd: process.cwd(), + }); + const daemonPort = await listenOnLoopback(daemon); + const proxy = createDaemonProxyServer({ + upstreamBaseUrl: `http://127.0.0.1:${daemonPort}`, + upstreamToken: 'daemon-secret', + clientToken: 'client-secret', + }); + let tracked = ''; + try { + const origin = `http://127.0.0.1:${await listenOnLoopback(proxy)}`; + const content = Buffer.from('ABCDEF'); + const headers = { + authorization: 'Bearer client-secret', + 'content-type': 'application/json', + }; + const preflight = await fetch(`${origin}/upload/preflight`, { + method: 'POST', + headers, + body: JSON.stringify({ + uploadAttemptId: crypto.randomUUID(), + sha256: crypto.createHash('sha256').update(content).digest('hex'), + fileName: 'demo.apk', + sizeBytes: content.length, + artifactType: 'file', + platform: 'android', + }), + }); + expect(preflight.status).toBe(200); + const ticket = (await preflight.json()) as { + uploadId: string; + upload: { url: string; headers: Record }; + }; + expect(new URL(ticket.upload.url).origin).toBe(origin); + expect(ticket.upload.headers.authorization).toBe('Bearer client-secret'); + const put = (range: string, body: string) => + fetch(ticket.upload.url, { + method: 'PUT', + headers: { ...ticket.upload.headers, 'content-range': range }, + body, + }); + const first = await put('bytes 0-1/6', 'AB'); + expect(first.status).toBe(308); + expect(first.headers.get('x-upload-offset')).toBe('2'); + await first.text(); + if (mode === 'disconnect and resume') { + const prefix = deferred(); + const rolledBack = deferred(); + let payload = ''; + let watcher: fs.FSWatcher | undefined; + const createWriter = fs.createWriteStream; + const writerSpy = vi.spyOn(fs, 'createWriteStream').mockImplementation((...args) => { + const writer = Reflect.apply(createWriter, fs, args) as fs.WriteStream; + payload = String(args[0]); + writer.once('open', () => { + watcher = fs.watch(payload, () => { + void fs.promises.readFile(payload, 'utf8').then((bytes) => { + if (bytes === 'ABCD') prefix.resolve(bytes); + }, prefix.reject); + }); + }); + return writer; + }); + const truncate = fs.promises.truncate; + const rollbackSpy = vi + .spyOn(fs.promises, 'truncate') + .mockImplementation(async (path, length) => { + await truncate(path, length); + if (String(path) === payload && length === 2) rolledBack.resolve(); + }); + onTestFinished(() => { + watcher?.close(); + writerSpy.mockRestore(); + rollbackSpy.mockRestore(); + }); + const client = http.request(ticket.upload.url, { + method: 'PUT', + headers: { ...ticket.upload.headers, 'content-range': 'bytes 2-5/6' }, + }); + const errors: Error[] = []; + client.on('error', (error) => errors.push(error)); + onTestFinished(() => { + client.destroy(); + }); + client.write('CD'); + expect(await prefix.promise).toBe('ABCD'); + assert.equal(fs.readFileSync(payload, 'utf8'), 'ABCD'); + const closed = new Promise((resolve) => client.once('close', () => resolve())); + client.destroy(); + await closed; + await rolledBack.promise; + expect(fs.readFileSync(payload, 'utf8')).toBe('AB'); + expect(client.writableEnded).toBe(false); + expect( + errors.every((error) => (error as NodeJS.ErrnoException).code === 'ECONNRESET'), + ).toBe(true); + watcher?.close(); + writerSpy.mockRestore(); + rollbackSpy.mockRestore(); + const probe = await put('bytes 0-1/6', 'AB'); + expect(probe.status).toBe(308); + expect(probe.headers.get('x-upload-offset')).toBe('2'); + await probe.text(); + } + const complete = await put('bytes 2-5/6', 'CDEF'); + expect(complete.status).toBe(200); + await complete.text(); + const finalize = await fetch(`${origin}/upload/finalize`, { + method: 'POST', + headers, + body: JSON.stringify({ uploadId: ticket.uploadId }), + }); + expect(finalize.status).toBe(200); + tracked = ((await finalize.json()) as { uploadId: string }).uploadId; + expect(fs.readFileSync(prepareUploadedArtifact(tracked))).toEqual(content); + } finally { + if (tracked) cleanupUploadedArtifact(tracked); + await closeLoopbackServer(proxy); + await closeLoopbackServer(daemon); + } + }); + } finally { + runtime.dispose(); + } + }, +); diff --git a/packages/host-kit/src/internal/transport-phases.test.ts b/packages/host-kit/src/internal/transport-phases.test.ts index 68865afe46..fafe9784cf 100644 --- a/packages/host-kit/src/internal/transport-phases.test.ts +++ b/packages/host-kit/src/internal/transport-phases.test.ts @@ -207,6 +207,38 @@ test('disposing the phase observer removes only its own listeners', () => { assert.equal(fake.listenerCount('response'), 0); }); +test('suspend resumes counting when the same phase is armed again', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 10_000); + watchdog.suspend(); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 3); + assert.equal(watchdog.signal.aborted, false); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'response-body', 120_000); +}); + +test('body progress during a suspend cannot restart the stopped ceiling', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + watchdog.suspend(); + watchdog.touchBody(); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 2); + assert.equal(watchdog.signal.aborted, false); +}); + +test('caller cancellation still binds an exchange whose idle ceiling is suspended', async () => { + const caller = new AbortController(); + const watchdog = watchPhases(caller.signal); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + watchdog.suspend(); + const cancellation = new Error('caller went away'); + caller.abort(cancellation); + assert.equal(watchdog.signal.aborted, true); + assert.equal(watchdog.signal.reason, cancellation); +}); + function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); assert.equal(isTransferPhaseTimeoutError(reason), true); diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index d698985b1e..165236b05e 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -160,6 +160,9 @@ export type TransferPhaseWatchdog = { signal: AbortSignal; arm(phase: TransferPhase, timeoutMs: number): void; touchBody(): void; + /** Stops counting an idle ceiling while the holder waits on demand, keeping its armed phase. */ + suspend(): void; + /** Stops observing this attempt. A suspended phase stays suspended. */ dispose(): void; }; @@ -168,11 +171,12 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; let armed: { phase: TransferPhase; timeoutMs: number } | undefined; let timer: ReturnType | undefined; + let suspended = false; const rearm = () => { if (timer !== undefined) clearTimeout(timer); timer = undefined; const current = armed; - if (!current) return; + if (!current || suspended) return; timer = setTimeout(() => { timer = undefined; phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); @@ -183,6 +187,7 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe signal, arm(nextPhase, timeoutMs) { if (signal.aborted) return; + suspended = false; armed = { phase: nextPhase, timeoutMs }; rearm(); }, @@ -190,6 +195,11 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe if (!armed || !BODY_PHASES.has(armed.phase)) return; rearm(); }, + suspend() { + suspended = true; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, dispose() { armed = undefined; if (timer !== undefined) clearTimeout(timer); diff --git a/packages/host-kit/src/transport.ts b/packages/host-kit/src/transport.ts index 16202f2ff3..eefbc20b97 100644 --- a/packages/host-kit/src/transport.ts +++ b/packages/host-kit/src/transport.ts @@ -8,6 +8,9 @@ export { readNodeHttpRequestBody, readNodeHttpResponseBody, timingSafeStringEqual, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + type TransferPhaseWatchdog, type NodeHttpRequester, } from './internal/transport.ts'; diff --git a/packages/proxy/README.md b/packages/proxy/README.md index 94b28e743c..f5042d9a8b 100644 --- a/packages/proxy/README.md +++ b/packages/proxy/README.md @@ -94,10 +94,46 @@ Your Node HTTP client must send exactly one valid `Host` header on every request configured. Configure raw-socket and HTTP/1.0 health probes to send `Host: localhost` (or your deployment's host and port). +### How the proxy reaches the daemon + +On artifact transfer routes — `POST /upload`, `/upload/preflight` and `/upload/finalize`, +`PUT /upload/direct/*` and `GET /artifacts*` — the proxy forwards with `node:http` / `node:https`. +Each transfer has a connect limit of 30 seconds, a response-header wait of 30 seconds that +starts only once its own outgoing body has finished and the connection is established, and a +120-second limit on body inactivity. For downloads, that limit counts time spent waiting for +upstream bytes while you read the response; it pauses when you stop reading. Every forwarded +exchange also ends at 600 seconds regardless of how steadily its bytes move. Your `upstreamTimeoutMs` +is an additional cap. The earliest caller cancellation, operator cap, phase limit or aggregate expiry +ends the exchange. A transfer does not retry or restart its aggregate. + +Transfer phase and aggregate timeout error responses use HTTP 500 and include `reason`, +`transferPhase` and `timeoutMs`. These fields appear only for `COMMAND_FAILED` with +`reason: transfer_phase_timeout` and the matching connect/header (30000ms) or body (120000ms) +phase budget, or `reason: transfer_forwarding_deadline_exceeded` with +`transferPhase: forwarding` and `timeoutMs: 600000`. Other errors omit these fields. +A timeout while you consume an accepted response fails the body read instead of replacing its status. + +On these routes: + +- Redirects are returned without being followed. The daemon's `308` resume response keeps its + `Range` and `x-upload-offset` headers. +- Default request headers added by `fetch`, such as `accept` and `user-agent`, are not sent. +- A patched global `fetch` or an `undici` global dispatcher no longer intercepts the request. + +Supply `upstreamFetch` if you need those Fetch behaviors or your own routing. + +Environment-proxy routing depends on your runtime. In loopback checks, Node 22.12 ignores +`HTTP_PROXY` and `HTTPS_PROXY` for both transports. Node 26.11.1 uses them when +`NODE_USE_ENV_PROXY=1`. Bun 1.4.2 Fetch uses them without that opt-in, while the default transfer +transport requires it. On Node 26.11.1 and Bun 1.4.2, `NO_PROXY=127.0.0.1` bypasses the proxy. +If you use Bun and rely on Fetch's proxy routing, set `NODE_USE_ENV_PROXY=1` or supply `upstreamFetch`. + +By default, `/rpc`, `/health` and request diagnostics use global `fetch`. + ### Reach the daemon over your own transport -By default the proxy calls the daemon with the global `fetch`. Pass `upstreamFetch` when the daemon -is only reachable some other way, such as a tunnel the device host opened to your gateway: +Pass `upstreamFetch` when the daemon is only reachable some other way, such as a tunnel the device +host opened to your gateway: ```ts const proxy = createDaemonProxy({ @@ -111,6 +147,16 @@ const proxy = createDaemonProxy({ `upstreamFetch` receives a `Request` addressed to `upstreamBaseUrl` and must resolve with the daemon's `Response`. Abort the exchange when `request.signal` aborts. +On transfer routes, the proxy bounds its wait for your transport and response body. It settles on +your abort, your `upstreamTimeoutMs`, its 600-second aggregate and its demand-aware response-body +inactivity limit. Work your transport starts can keep running if it ignores `request.signal`. The +proxy does not wait for that work or for a cancellation callback that never settles. It cannot +measure a connect or a header wait it does not own, so on transfer routes your transport keeps those +obligations. Bound connection establishment to 30 seconds and the response-header wait to 30 seconds +once the connection is established and the outgoing body has finished. Stream `request.body` with +120-second body inactivity limits. Stop your request and response sources on cancellation. A body +pull alone does not confirm that an outgoing byte was sent. + ### Authenticate each client yourself One `clientToken` is shared by every client. To give each client its own credential, pass @@ -158,9 +204,9 @@ authenticate: async (request) => { | `upstreamToken` | required | The daemon's auth token. | | `clientToken` | required without `authenticate` | Token clients must send as their daemon auth token. | | `authenticate` | none | Authenticates each client and names its tenant; replaces `clientToken`. | -| `upstreamFetch` | global `fetch` | Sends one request to the daemon. | +| `upstreamFetch` | none | Sends one request to the daemon. Without it, transfer routes use `node:http(s)` and every other route uses the global `fetch`. | | `maxRpcBodyBytes` | 64 MiB, the daemon's own request limit | Larger `/rpc` bodies get a 400 response. | -| `upstreamTimeoutMs` | none | Longest time one daemon request may take, including its body. Without it, a request lasts as long as the client waits, and the client's command timeout bounds it. | +| `upstreamTimeoutMs` | none | Independent cap on one daemon request, including its body. Without it, a request lasts as long as the client waits, bounded by the client's command timeout and, on transfer routes, the 600-second forwarding aggregate. | `maxRpcBodyBytes` limits the bytes of one `/rpc` body. The proxy reads the whole body before it authenticates the client, then decodes and parses it. One unauthenticated request can therefore diff --git a/packages/proxy/src/daemon-proxy-transfer.test.ts b/packages/proxy/src/daemon-proxy-transfer.test.ts new file mode 100644 index 0000000000..13af18a45c --- /dev/null +++ b/packages/proxy/src/daemon-proxy-transfer.test.ts @@ -0,0 +1,408 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createDaemonProxy, + createDaemonProxyServer, + type DaemonProxyOptions, +} from './daemon-proxy.ts'; + +const options = { + upstreamBaseUrl: 'http://daemon.internal:4310', + upstreamToken: 'daemon-secret', + clientToken: 'client-secret', +}; + +async function listen(server: http.Server): Promise { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function requestThroughProxy(origin: string, path: string, method = 'POST') { + return new Promise<{ status: number; body: Record }>((resolve, reject) => { + const req = http.request( + `${origin}${path}`, + { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + }, + (res) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + text += chunk; + }); + res.on('error', reject); + res.on('end', () => + resolve({ status: res.statusCode!, body: text ? JSON.parse(text) : {} }), + ); + }, + ); + req.on('error', reject); + req.end(method === 'POST' ? '{}' : undefined); + }); +} + +test('the actual proxy HTTP response exposes a header timeout as 500 with exactly three transfer details', async () => { + let entered!: () => void; + const uploaded = new Promise((resolve) => { + entered = resolve; + }); + const daemon = await listen( + http.createServer((req) => { + req.resume(); + req.once('end', entered); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const original = http.request; + let flushed!: (request: http.ClientRequest) => void; + const finished = new Promise((resolve) => { + flushed = resolve; + }); + const observer = vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(original, http, args) as http.ClientRequest; + if (req.getHeader('authorization') === 'Bearer daemon-secret') { + req.once('finish', () => flushed(req)); + } + return req; + }); + onTestFinished(() => observer.mockRestore()); + const response = requestThroughProxy(origin, '/upload/preflight'); + await uploaded; + expect((await finished).writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(30_000); + expect(await response).toEqual({ + status: 500, + body: { + ok: false, + error: 'Artifact transfer response-headers timed out', + code: 'COMMAND_FAILED', + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }, + }); +}); + +test('the actual HTTP response exposes the forwarding aggregate when a custom transport ignores cancellation', async () => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamFetch: () => { + entered(); + return new Promise(() => undefined); + }, + }), + ); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const response = requestThroughProxy(origin, '/upload/preflight'); + await started; + await vi.advanceTimersByTimeAsync(600_000); + expect(await response).toEqual({ + status: 500, + body: { + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }, + }); +}); + +test.each([ + { + code: 'INVALID_ARGS', + status: 400, + phase: 'response-body', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'unknown-phase', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'response-headers', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'forwarding', + budget: 600_001, + reason: 'transfer_forwarding_deadline_exceeded', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'response-body', + budget: 600_000, + reason: 'transfer_forwarding_deadline_exceeded', + }, +] as const)( + 'actual HTTP $status omits invalid transfer detail combinations ($code/$phase/$budget)', + async ({ code, status, phase, budget, reason }) => { + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamFetch: async () => { + throw new AppError(code, 'Invalid transfer metadata', { + reason, + transferPhase: phase, + timeoutMs: budget, + secret: 'hidden', + }); + }, + }), + ); + expect(await requestThroughProxy(origin, '/upload/preflight')).toEqual({ + status, + body: { ok: false, error: 'Invalid transfer metadata', code }, + }); + }, +); + +test('recognized forwarding details are additive on transfers and absent on diagnostics and health', async () => { + const proxy = createDaemonProxy({ + ...options, + upstreamFetch: async () => { + throw new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + secret: 'hidden', + }); + }, + }); + const send = (path: string) => + proxy.handle( + new Request(`http://gateway.test${path}`, { + headers: { authorization: 'Bearer client-secret' }, + }), + ); + expect(await (await send('/artifacts/build')).json()).toEqual({ + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + for (const path of ['/health', '/sessions/session/requests/request/diagnostics']) { + expect(await (await send(path)).json()).toEqual({ + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + }); + } +}); + +test('a custom transport receives every route family including transfers', async () => { + const seen: Request[] = []; + const custom: DaemonProxyOptions['upstreamFetch'] = async (request) => { + seen.push(request); + const path = new URL(request.url).pathname; + if (path === '/upload/preflight') + return Response.json({ + uploadId: 'ticket', + upload: { url: 'http://daemon.internal:4310/upload/direct/ticket' }, + }); + if (path === '/rpc') return Response.json({ jsonrpc: '2.0', id: 1, result: { ok: true } }); + return new Response('custom answer'); + }; + const proxy = createDaemonProxy({ ...options, upstreamFetch: custom }); + for (const [method, path] of [ + ['POST', '/upload'], + ['POST', '/upload/preflight'], + ['POST', '/upload/finalize'], + ['PUT', '/upload/direct/ticket'], + ['GET', '/artifacts/build'], + ['GET', '/sessions/session/requests/request/diagnostics'], + ['GET', '/health'], + ['POST', '/rpc'], + ]) { + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + ...(method === 'POST' || method === 'PUT' + ? { + body: + path === '/rpc' + ? JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'agent-device.command', + params: { token: 'client-secret', command: 'version' }, + }) + : '{}', + } + : {}), + }), + ); + expect(response.status, path).toBe(200); + await response.text(); + } + expect(seen.map((request) => new URL(request.url).pathname)).toEqual([ + '/upload', + '/upload/preflight', + '/upload/finalize', + '/upload/direct/ticket', + '/artifacts/build', + '/sessions/session/requests/request/diagnostics', + '/health', + '/rpc', + ]); + expect( + seen.every((request) => request.headers.get('authorization') === 'Bearer daemon-secret'), + ).toBe(true); +}); + +test('an explicit operator cap settles an ignored custom request and reaches its bounded signal', async () => { + let bounded!: Request; + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamTimeoutMs: 10, + upstreamFetch: (request) => { + bounded = request; + return new Promise(() => undefined); + }, + }), + ); + const response = await requestThroughProxy(origin, '/upload/preflight'); + expect(response.status).toBe(500); + assert.ok(bounded.signal.aborted); + expect(bounded.signal.reason.name).toBe('TimeoutError'); + expect(response.body).not.toHaveProperty('transferPhase'); + expect(response.body).not.toHaveProperty('timeoutMs'); +}); + +test.each([401, 413, 200])( + 'an early %s crosses the real proxy HTTP listener while ingress remains unfinished', + async (status) => { + const daemon = await listen( + http.createServer((_req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end('{"answer":"intact"}'); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + const response = await new Promise<{ status: number; text: string }>((resolve, reject) => { + const req = http.request( + `${origin}/upload/direct/ticket`, + { + method: 'PUT', + headers: { + authorization: 'Bearer client-secret', + 'content-type': 'application/octet-stream', + }, + }, + (res) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + text += chunk; + }); + res.once('error', reject); + res.once('end', () => { + expect(req.writableEnded).toBe(false); + resolve({ status: res.statusCode!, text }); + req.destroy(); + }); + }, + ); + req.on('error', reject); + req.write(Buffer.from('unfinished ingress')); + onTestFinished(() => { + req.destroy(); + }); + }); + expect(response).toEqual({ status, text: '{"answer":"intact"}' }); + }, +); + +test.each([205, 304, 400, 500, 599])( + 'a final upstream %s crosses the real HTTP proxy without normalization', + async (status) => { + const noBody = status === 205 || status === 304; + const daemon = await listen( + http.createServer((_req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(noBody ? undefined : '{"answer":"upstream"}'); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + expect(await requestThroughProxy(origin, '/artifacts/build', 'GET')).toEqual({ + status, + body: noBody ? {} : { answer: 'upstream' }, + }); + }, +); + +test.each([ + { transport: 'node', status: 204 }, + { transport: 'node', status: 205 }, + { transport: 'node', status: 304 }, + { transport: 'custom', status: 204 }, + { transport: 'custom', status: 205 }, + { transport: 'custom', status: 304 }, +])( + 'preflight preserves null-body $status and request identity through $transport transport', + async ({ transport, status }) => { + const headers = { 'x-request-id': 'daemon-preflight-request' }; + const daemon = await listen( + http.createServer((req, res) => { + req.resume(); + res.writeHead(status, headers); + res.end(); + }), + ); + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamBaseUrl: daemon, + ...(transport === 'custom' + ? { upstreamFetch: async () => new Response(null, { status, headers }) } + : {}), + }), + ); + const response = await fetch(`${origin}/upload/preflight`, { + method: 'POST', + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + body: '{}', + redirect: 'error', + }); + expect(response.status).toBe(status); + expect(response.headers.get('x-request-id')).toBe('daemon-preflight-request'); + expect(await response.text()).toBe(''); + }, +); diff --git a/packages/proxy/src/daemon-proxy.ts b/packages/proxy/src/daemon-proxy.ts index 38c6c897ae..76c70182f5 100644 --- a/packages/proxy/src/daemon-proxy.ts +++ b/packages/proxy/src/daemon-proxy.ts @@ -25,9 +25,14 @@ import { } from './install-source-admission.ts'; /** - * Carries one request to the upstream daemon and resolves with its response. The default is the - * global `fetch`; supply your own to reach the daemon over another transport, such as a - * WebSocket tunnel. Abort the exchange when `request.signal` aborts. + * Carries one request to the upstream daemon and resolves with its response. Without `upstreamFetch` + * the proxy reaches the daemon over `node:http(s)` on artifact transfer routes and over the global + * `fetch` on `/rpc`, health and diagnostics. Supply your own to reach the daemon over another + * transport, such as a WebSocket tunnel. Abort the exchange when `request.signal` aborts. + * + * A custom transport owns its phase guarantees and must observe `request.signal` on every route. + * On transfer routes, the proxy bounds its own response and body waits, even if the transport + * ignores cancellation. It cannot forcibly join work started by that transport. */ export type DaemonProxyUpstreamFetch = (request: Request) => Promise; @@ -60,10 +65,12 @@ export type DaemonProxyOptions = { publicOrigin?: string; maxRpcBodyBytes?: number; /** - * Longest time one daemon request may take, including its body. Unset, an exchange lasts as long - * as its client request: the client's command envelope bounds it. + * Independent cap on one upstream request, including its response body. Transfers also observe + * their phase limits and 600-second aggregate. When unset, other requests use the client's + * command envelope. The health probe keeps its separate five-second limit. */ upstreamTimeoutMs?: number; + /** Overrides the transport for every upstream route. See `DaemonProxyUpstreamFetch`. */ upstreamFetch?: DaemonProxyUpstreamFetch; } & ( | { @@ -102,6 +109,7 @@ type NormalizedProxyOptions = { maxRpcBodyBytes: number; upstreamTimeoutMs: number | undefined; upstreamFetch: DaemonProxyUpstreamFetch; + upstreamTransport: (route: string, request: Request) => Promise; }; /** The client a request was authenticated as; `tenantId` is set only by `authenticate`. */ @@ -141,7 +149,7 @@ export function createDaemonProxy(options: DaemonProxyOptions): DaemonProxy { try { return await handleProxyRequest(request, normalized, instanceId); } catch (error) { - return proxyErrorResponse(error); + return proxyErrorResponse(error, request); } }, }; @@ -349,7 +357,8 @@ async function forwardProxyRequest(params: { }): Promise { const { request, route, options, rpcBody, caller } = params; const body = resolveUpstreamBody(request, route, rpcBody, options.upstreamToken, caller.tenantId); - const upstreamResponse = await options.upstreamFetch( + const upstreamResponse = await options.upstreamTransport( + route, new Request(buildUpstreamUrl(options.upstreamBaseUrl, route, request.url), { method: request.method, headers: buildUpstreamHeaders(request.headers, options.upstreamToken, route, caller.tenantId), @@ -371,10 +380,6 @@ async function forwardProxyRequest(params: { return new Response(upstreamResponse.body, { status: upstreamResponse.status, headers }); } -/** - * An upstream exchange lives exactly as long as the client keeps waiting for it: the daemon - * turns a dropped upstream request into cancellation of in-flight runner work. - */ function upstreamSignal(request: Request, options: NormalizedProxyOptions): AbortSignal { if (options.upstreamTimeoutMs === undefined) return request.signal; return AbortSignal.any([request.signal, AbortSignal.timeout(options.upstreamTimeoutMs)]); @@ -409,6 +414,13 @@ async function rewrittenUploadPreflightResponse(params: { publicOrigin: string | undefined; }): Promise { const { request, upstreamResponse, headers, uploadHeaders, publicOrigin } = params; + if ( + upstreamResponse.status === 204 || + upstreamResponse.status === 205 || + upstreamResponse.status === 304 + ) { + return new Response(null, { status: upstreamResponse.status, headers }); + } const text = await upstreamResponse.text(); headers.set('content-type', upstreamResponse.headers.get('content-type') ?? 'application/json'); return new Response( @@ -516,6 +528,8 @@ function normalizeProxyOptions(options: DaemonProxyOptions): NormalizedProxyOpti const clientToken = options.authenticate ? undefined : normalizeToken(options.clientToken ?? '', 'clientToken'); + const upstreamFetch = options.upstreamFetch ?? ((request) => fetch(request)); + const customUpstreamFetch = options.upstreamFetch; return { upstreamBaseUrl, publicOrigin: @@ -527,7 +541,15 @@ function normalizeProxyOptions(options: DaemonProxyOptions): NormalizedProxyOpti authenticate: options.authenticate, maxRpcBodyBytes: options.maxRpcBodyBytes ?? DEFAULT_MAX_RPC_BODY_BYTES, upstreamTimeoutMs: options.upstreamTimeoutMs, - upstreamFetch: options.upstreamFetch ?? ((request) => fetch(request)), + upstreamFetch, + upstreamTransport: async (route, request) => { + const { isTransferRoute, createNodeTransferFetch, settleUpstreamFetch } = + await import('./transfer-transport.ts'); + if (!isTransferRoute(route, request.method)) return upstreamFetch(request); + return customUpstreamFetch + ? settleUpstreamFetch(customUpstreamFetch, request) + : createNodeTransferFetch()(request); + }, }; } @@ -864,10 +886,17 @@ function rpcErrorResponse(rpcId: unknown, error: unknown): Response { ); } -function proxyErrorResponse(error: unknown): Response { +async function proxyErrorResponse(error: unknown, request: Request): Promise { + const { isTransferRoute, transferWireDetails } = await import('./transfer-transport.ts'); + const transfer = isTransferRoute(resolveProxyRoute(request.url), request.method); const normalized = normalizeError(error); return Response.json( - { ok: false, error: normalized.message, code: normalized.code }, + { + ok: false, + error: normalized.message, + code: normalized.code, + ...(transfer ? transferWireDetails(error) : {}), + }, { status: normalized.code === 'INVALID_ARGS' ? 400 : 500 }, ); } diff --git a/packages/proxy/src/transfer-transport-lifetime.test.ts b/packages/proxy/src/transfer-transport-lifetime.test.ts new file mode 100644 index 0000000000..815286ddc7 --- /dev/null +++ b/packages/proxy/src/transfer-transport-lifetime.test.ts @@ -0,0 +1,803 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import http, { type ClientRequest, type ServerResponse } from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { createNodeTransferFetch, settleUpstreamFetch } from './transfer-transport.ts'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +function controlledClock() { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); +} + +async function upstream(handler: http.RequestListener) { + const server = http.createServer(handler); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function observeRequest(inspect?: (request: ClientRequest) => void) { + const original = http.request; + const made = deferred(); + const closed = deferred(); + const spy = vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(original, http, args) as ClientRequest; + req.once('close', () => closed.resolve()); + inspect?.(req); + made.resolve(req); + return req; + }); + onTestFinished(() => spy.mockRestore()); + return { made: made.promise, closed: closed.promise }; +} + +function phaseFailure(error: unknown, phase: string, timeoutMs: number) { + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs, + }); +} + +function emptyChunkFlood(caller: AbortController, reason: Error) { + const limit = 1_024; + let reads = 0; + let readsAtAbort = limit; + let timer: ReturnType | undefined; + const body = new ReadableStream( + { + pull(controller) { + if (reads === 0) { + timer = setTimeout(() => { + readsAtAbort = reads; + caller.abort(reason); + }, 0); + } + if (reads === limit) controller.close(); + else { + reads += 1; + controller.enqueue(new Uint8Array()); + } + }, + }, + { highWaterMark: 0 }, + ); + onTestFinished(() => clearTimeout(timer)); + return { body, limit, readsAtAbort: () => readsAtAbort }; +} + +test('empty upload chunks yield to caller cancellation before the bounded flood ends', async () => { + const origin = await upstream((req) => req.resume()); + const observed = observeRequest(); + const caller = new AbortController(); + const reason = new Error('cancel upload while empty chunks are ready'); + const flood = emptyChunkFlood(caller, reason); + const pending = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body: flood.body, + signal: caller.signal, + duplex: 'half', + } as RequestInit), + ); + await expect(pending).rejects.toBe(reason); + await observed.closed; + expect(flood.readsAtAbort()).toBeLessThan(flood.limit); + expect(flood.body.locked).toBe(false); +}); + +test('empty custom response chunks yield to caller cancellation before the bounded flood ends', async () => { + const caller = new AbortController(); + const reason = new Error('cancel response while empty chunks are ready'); + const flood = emptyChunkFlood(caller, reason); + const response = await settleUpstreamFetch( + async () => new Response(flood.body), + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + await expect(response.arrayBuffer()).rejects.toBe(reason); + expect(flood.readsAtAbort()).toBeLessThan(flood.limit); + expect(flood.body.locked).toBe(false); +}); + +test('finite empty chunks preserve the following upload and custom response bytes', async () => { + const emptyPrefix = () => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array()); + controller.enqueue(new Uint8Array()); + controller.enqueue(new TextEncoder().encode('payload intact')); + controller.close(); + }, + }); + const origin = await upstream((req, res) => { + let text = ''; + req.setEncoding('utf8'); + req.on('data', (chunk: string) => { + text += chunk; + }); + req.on('end', () => res.end(text)); + }); + const uploaded = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body: emptyPrefix(), + duplex: 'half', + } as RequestInit), + ); + expect(await uploaded.text()).toBe('payload intact'); + const custom = await settleUpstreamFetch( + async () => new Response(emptyPrefix()), + new Request('http://daemon.internal/artifacts/build'), + ); + expect(await custom.text()).toBe('payload intact'); +}); + +test.each([401, 413, 200, 204])( + 'an early %s stops and unlocks an endless incoming reader before disposal', + async (status) => { + const origin = await upstream((_req, res) => { + res.writeHead(status, { 'content-type': 'text/plain' }); + res.end(status === 204 ? undefined : 'accepted answer'); + }); + controlledClock(); + const observed = observeRequest(); + const waiting = deferred(); + let cancelled = false; + const body = new ReadableStream( + { + start(controller) { + controller.enqueue(new Uint8Array([1])); + }, + pull() { + waiting.resolve(); + return new Promise(() => undefined); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ); + const response = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ); + await waiting.promise; + expect(response.status).toBe(status); + expect(await response.text()).toBe(status === 204 ? '' : 'accepted answer'); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(body.locked).toBe(false); + expect(cancelled).toBe(false); + expect((await observed.made).closed).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test('a progressing upload gets a fresh header ceiling at actual finish, beyond thirty seconds', async () => { + const first = deferred(); + const second = deferred(); + const receivedEnd = deferred(); + let bytes = 0; + const origin = await upstream((req) => { + req.on('data', (chunk: Buffer) => { + bytes += chunk.length; + if (bytes >= 1) first.resolve(); + if (bytes >= 2) second.resolve(); + }); + req.once('end', () => receivedEnd.resolve()); + }); + controlledClock(); + const observed = observeRequest(); + let producer!: ReadableStreamDefaultController; + const body = new ReadableStream( + { + start(controller) { + producer = controller; + }, + }, + { highWaterMark: 0 }, + ); + let settled = false; + const result = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + producer.enqueue(new Uint8Array([1])); + await first.promise; + await vi.advanceTimersByTimeAsync(31_000); + expect(settled).toBe(false); + producer.enqueue(new Uint8Array([2])); + await second.promise; + const req = await observed.made; + const finished = once(req, 'finish'); + producer.close(); + await finished; + await receivedEnd.promise; + expect(req.writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(29_999); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(req.destroyed).toBe(true); + phaseFailure(await result, 'response-headers', 30_000); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); +}); + +test('a bodyless request starts its header ceiling after actual finish on a connected socket', async () => { + const entered = deferred(); + const origin = await upstream(() => entered.resolve()); + controlledClock(); + const observed = observeRequest(); + const result = createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)).catch( + (error: unknown) => error, + ); + await entered.promise; + expect((await observed.made).writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(30_000); + expect((await observed.made).destroyed).toBe(true); + phaseFailure(await result, 'response-headers', 30_000); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); +}); + +test('response inactivity charges a pending producer, then joins its owned request close', async () => { + const origin = await upstream((_req, res) => res.flushHeaders()); + controlledClock(); + const observed = observeRequest(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + const result = response.text().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(119_999); + expect((await observed.made).destroyed).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect((await observed.made).destroyed).toBe(true); + phaseFailure(await result, 'response-body', 120_000); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(vi.getTimerCount()).toBe(0); +}); + +test('downstream backpressure suspends response inactivity without suspending the aggregate', async () => { + const peer = deferred(); + const origin = await upstream((_req, res) => { + res.flushHeaders(); + peer.resolve(res); + }); + controlledClock(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + await vi.advanceTimersByTimeAsync(240_000); + (await peer.promise).end('available on demand'); + expect(await response.text()).toBe('available on demand'); + expect(vi.getTimerCount()).toBe(0); +}); + +test('continuous response progress cannot refresh the six hundred second aggregate', async () => { + const peer = deferred(); + const origin = await upstream((_req, res) => { + res.flushHeaders(); + peer.resolve(res); + }); + controlledClock(); + const observed = observeRequest(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + const reader = response.body!.getReader(); + const res = await peer.promise; + for (let index = 0; index < 5; index += 1) { + const reading = reader.read(); + await vi.advanceTimersByTimeAsync(110_000); + res.write(Buffer.from([index + 1])); + const chunk = await reading; + expect(chunk.done).toBe(false); + expect(Array.from(chunk.value!)).toEqual([index + 1]); + } + const failure = reader.read().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(50_000); + expect((await observed.made).destroyed).toBe(true); + const error = await failure; + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['caller', 'idle', 'aggregate'] as const)( + 'ignored foreign read and cancel cannot hold proxy settlement after %s expiry', + async (expiry) => { + controlledClock(); + const caller = new AbortController(); + const reading = deferred(); + const cancelled = deferred(); + const lateRead = deferred(); + const lateCancel = deferred(); + const foreign = new ReadableStream( + { + pull() { + reading.resolve(); + return lateRead.promise; + }, + cancel(reason) { + cancelled.resolve(reason); + return lateCancel.promise; + }, + }, + { highWaterMark: 0 }, + ); + let bounded: Request | undefined; + const response = await settleUpstreamFetch( + async (request) => { + bounded = request; + return new Response(foreign); + }, + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + if (expiry === 'aggregate') await vi.advanceTimersByTimeAsync(500_000); + let settled = false; + const result = response.text().catch((error: unknown) => error); + void result.then(() => { + settled = true; + }); + await reading.promise; + const callerReason = new AppError('COMMAND_FAILED', 'Caller stopped waiting', { + reason: 'client_gone', + }); + if (expiry === 'caller') caller.abort(callerReason); + else if (expiry === 'idle') await vi.advanceTimersByTimeAsync(120_000); + else await vi.advanceTimersByTimeAsync(100_000); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); + const error = await result; + if (expiry === 'idle') phaseFailure(error, 'response-body', 120_000); + else if (expiry === 'caller') expect(error).toBe(callerReason); + else { + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + } + expect(await cancelled.promise).toBe(error); + expect(bounded?.signal.reason).toBe(error); + expect(foreign.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + lateRead.reject(new Error('late foreign producer rejection')); + lateCancel.reject(new Error('late foreign cancel rejection')); + await new Promise((resolve) => setImmediate(resolve)); + }, +); + +test('the aggregate cancels an undemanded foreign body without joining its opaque cancel', async () => { + controlledClock(); + const cancelled = deferred(); + const response = await settleUpstreamFetch( + async () => + new Response( + new ReadableStream( + { + cancel(reason) { + cancelled.resolve(reason); + return new Promise(() => undefined); + }, + }, + { highWaterMark: 0 }, + ), + ), + new Request('http://daemon.internal/artifacts/build'), + ); + await vi.advanceTimersByTimeAsync(600_000); + const error = await response.text().catch((error: unknown) => error); + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + expect(await cancelled.promise).toBe(error); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['throw', 'reject'] as const)( + 'a synchronous custom %s releases all exchange timers', + async (kind) => { + controlledClock(); + const reason = new Error('custom transport failed'); + const pending = settleUpstreamFetch(() => { + if (kind === 'throw') throw reason; + return Promise.reject(reason); + }, new Request('http://daemon.internal/artifacts/build')); + await expect(pending).rejects.toBe(reason); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test('an already aborted request never starts foreign work or leaves a timer', async () => { + controlledClock(); + const reason = new Error('already stopped'); + let calls = 0; + await expect( + settleUpstreamFetch( + async () => { + calls += 1; + return new Response('unexpected'); + }, + new Request('http://daemon.internal/artifacts/build', { signal: AbortSignal.abort(reason) }), + ), + ).rejects.toBe(reason); + expect(calls).toBe(0); + expect(vi.getTimerCount()).toBe(0); +}); + +test('late custom response and rejection outcomes stay observed after cancellation', async () => { + const unhandled: unknown[] = []; + const rejected = (error: unknown) => { + unhandled.push(error); + }; + process.on('unhandledRejection', rejected); + onTestFinished(() => { + process.off('unhandledRejection', rejected); + }); + const caller = new AbortController(); + const late = deferred(); + const cancelled = deferred(); + const pending = settleUpstreamFetch( + () => late.promise, + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + const reason = new Error('stopped'); + caller.abort(reason); + await expect(pending).rejects.toBe(reason); + late.resolve( + new Response( + new ReadableStream({ + cancel() { + cancelled.resolve(); + }, + }), + ), + ); + await cancelled.promise; + const rejection = deferred(); + const other = new AbortController(); + const failed = settleUpstreamFetch( + () => rejection.promise, + new Request('http://daemon.internal/artifacts/build', { signal: other.signal }), + ); + other.abort(reason); + await expect(failed).rejects.toBe(reason); + rejection.reject(new Error('late tunnel failure')); + await new Promise((resolve) => setImmediate(resolve)); + expect(unhandled).toEqual([]); +}); + +test.each(['abort', 'peer-close', 'drain', 'idle', 'drain-only'] as const)( + 'an actual blocked write releases its drain wait on %s', + async (outcome) => { + const entered = deferred(); + const origin = await upstream((req, res) => { + req.pause(); + entered.resolve(req); + req.once('end', () => res.end('upload complete')); + }); + controlledClock(); + const blocked = deferred(); + let drains = 0; + const observed = observeRequest((req) => { + const onDrain = () => { + drains += 1; + }; + req.on('drain', onDrain); + onTestFinished(() => { + req.off('drain', onDrain); + }); + const write = req.write; + req.write = function (this: ClientRequest, ...args: unknown[]): boolean { + const accepted = Reflect.apply(write, this, args) as boolean; + if (!accepted) blocked.resolve(); + return accepted; + }; + }); + const caller = new AbortController(); + const body = new ReadableStream( + { + start(controller) { + controller.enqueue(new Uint8Array(32 * 1024 * 1024)); + controller.close(); + }, + }, + { highWaterMark: 0 }, + ); + const result = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + signal: caller.signal, + duplex: 'half', + } as RequestInit), + ).catch((error: unknown) => error); + await blocked.promise; + const incoming = await entered.promise; + await new Promise((resolve) => setImmediate(resolve)); + const req = await observed.made; + expect(req.writableNeedDrain).toBe(true); + expect(req.writableLength).toBeGreaterThan(0); + expect(req.listenerCount('drain')).toBe(2); // The probe and the adapter's actual wait. + let destroyedAtOriginalDeadline = false; + const reason = new Error('client left during backpressure'); + if (outcome === 'abort') caller.abort(reason); + else if (outcome === 'peer-close') incoming.destroy(); + else if (outcome === 'idle') await vi.advanceTimersByTimeAsync(120_000); + else if (outcome === 'drain-only') { + await vi.advanceTimersByTimeAsync(60_000); + req.emit('drain'); + expect(req.writableLength).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); + destroyedAtOriginalDeadline = req.destroyed; + await vi.advanceTimersByTimeAsync(120_000); + } else incoming.resume(); + const response = await result; + if (outcome === 'drain') { + assert.ok(response instanceof Response); + expect(await response.text()).toBe('upload complete'); + expect(drains).toBeGreaterThan(0); + } else if (outcome === 'abort') expect(response).toBe(reason); + else if (outcome === 'idle' || outcome === 'drain-only') + phaseFailure(response, 'request-body', 120_000); + else expect(response).toBeInstanceOf(Error); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(req.listenerCount('drain')).toBe(1); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + if (outcome === 'drain-only') expect(destroyedAtOriginalDeadline).toBe(true); + }, +); + +test('a completed exchange leaves its global-agent socket available to the next request', async () => { + const sockets: http.IncomingMessage['socket'][] = []; + const origin = await upstream((req, res) => { + sockets.push(req.socket); + res.end('pooled response'); + }); + const requests: ClientRequest[] = []; + observeRequest((req) => { + requests.push(req); + }); + const send = createNodeTransferFetch(); + expect(await (await send(new Request(`${origin}/artifacts/first`))).text()).toBe( + 'pooled response', + ); + expect(sockets[0]?.destroyed).toBe(false); + expect(await (await send(new Request(`${origin}/artifacts/second`))).text()).toBe( + 'pooled response', + ); + expect(requests[1]?.reusedSocket).toBe(true); + expect(sockets[1]).toBe(sockets[0]); + expect(sockets[1]?.destroyed).toBe(false); +}); + +test('a real TCP connection without a completed TLS handshake expires in the connect phase', async () => { + const accepted = deferred(); + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on('error', () => undefined); + socket.once('close', () => sockets.delete(socket)); + accepted.resolve(); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }); + controlledClock(); + const port = (server.address() as AddressInfo).port; + const result = createNodeTransferFetch()( + new Request(`https://127.0.0.1:${port}/artifacts/build`), + ).catch((error: unknown) => error); + await accepted.promise; + await vi.advanceTimersByTimeAsync(30_000); + phaseFailure(await result, 'connect', 30_000); + expect(vi.getTimerCount()).toBe(0); +}); + +test('downstream body cancellation aborts the custom request without joining an ignored cancel callback', async () => { + controlledClock(); + const reading = deferred(); + const cancelled = deferred(); + let bounded!: Request; + const body = new ReadableStream( + { + pull() { + reading.resolve(); + return new Promise(() => undefined); + }, + cancel(reason) { + cancelled.resolve(reason); + return new Promise(() => undefined); + }, + }, + { highWaterMark: 0 }, + ); + const response = await settleUpstreamFetch(async (request) => { + bounded = request; + return new Response(body); + }, new Request('http://daemon.internal/artifacts/build')); + const reader = response.body!.getReader(); + const pending = reader.read(); + await reading.promise; + const reason = new Error('downstream stopped reading'); + await reader.cancel(reason); + expect(await pending).toEqual({ done: true, value: undefined }); + expect(await cancelled.promise).toBe(reason); + expect(bounded.signal.reason).toBe(reason); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['iterator stop', 'request close', 'outgoing pump'] as const)( + 'owned cleanup awaits %s before settlement and timer disposal', + async (held) => { + const origin = await upstream((_req, res) => { + res.setHeader('connection', 'close'); + res.end('joined'); + }); + controlledClock(); + const stopEntered = deferred(); + const responseClosed = deferred(); + const pumpUnlocked = deferred(); + const pumpStopEntered = deferred(); + const allowPump = deferred(); + const allowStop = deferred(); + const closeEntered = deferred(); + const allowClose = deferred(); + onTestFinished(() => { + allowStop.resolve(); + allowClose.resolve(); + allowPump.resolve(); + }); + let unlocked = false; + const observed = observeRequest((req) => { + const once = req.once; + const closeRegistration = vi.spyOn(req, 'once').mockImplementation((event, listener) => { + if (event !== 'close') return Reflect.apply(once, req, [event, listener]); + closeRegistration.mockRestore(); + return Reflect.apply(once, req, [ + event, + (...args: unknown[]) => { + closeEntered.resolve(); + if (held === 'request close') + void allowClose.promise.then(() => Reflect.apply(listener, req, args)); + else Reflect.apply(listener, req, args); + }, + ]); + }); + onTestFinished(() => closeRegistration.mockRestore()); + req.prependListener('response', (res) => { + res.once('close', () => responseClosed.resolve()); + const iterate = res[Symbol.asyncIterator].bind(res); + res[Symbol.asyncIterator] = () => { + const iterator = iterate(); + const stop = iterator.return!.bind(iterator); + iterator.return = async (value) => { + const stopped = await stop(value); + stopEntered.resolve(); + if (held === 'iterator stop') await allowStop.promise; + return stopped; + }; + return iterator; + }; + }); + if (held === 'outgoing pump') queueMicrotask(() => req.flushHeaders()); + }); + const body = new ReadableStream( + { + start(controller) { + if (held !== 'outgoing pump') controller.enqueue(new Uint8Array([1])); + }, + }, + { highWaterMark: 0 }, + ); + const getReader = body.getReader.bind(body); + body.getReader = (() => { + const reader = getReader(); + const release = reader.releaseLock.bind(reader); + reader.releaseLock = () => { + release(); + unlocked = true; + pumpUnlocked.resolve(); + }; + if (held === 'outgoing pump') { + const add = AbortSignal.prototype.addEventListener; + const registration = vi + .spyOn(AbortSignal.prototype, 'addEventListener') + .mockImplementation(function (this: AbortSignal, event, listener, options) { + registration.mockRestore(); + expect(event).toBe('abort'); + return Reflect.apply(add, this, [ + event, + (value: Event) => { + pumpStopEntered.resolve(); + void allowPump.promise.then(() => { + if (typeof listener === 'function') listener.call(this, value); + else listener?.handleEvent(value); + }); + }, + options, + ]); + }); + onTestFinished(() => registration.mockRestore()); + } + return reader; + }) as typeof body.getReader; + const response = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ); + let settled = false; + const result = response.text().then((text) => { + settled = true; + return text; + }); + await { + 'iterator stop': stopEntered.promise, + 'request close': closeEntered.promise, + 'outgoing pump': pumpStopEntered.promise, + }[held]; + await Promise.all([observed.closed, responseClosed.promise]); + if (held !== 'outgoing pump') await pumpUnlocked.promise; + await new Promise((resolve) => setImmediate(resolve)); + expect(unlocked).toBe(held !== 'outgoing pump'); + expect(settled).toBe(false); + expect(vi.getTimerCount()).toBeGreaterThan(0); + allowStop.resolve(); + allowClose.resolve(); + allowPump.resolve(); + expect(await result).toBe('joined'); + await pumpUnlocked.promise; + expect(unlocked).toBe(true); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); + }, +); diff --git a/packages/proxy/src/transfer-transport.test.ts b/packages/proxy/src/transfer-transport.test.ts new file mode 100644 index 0000000000..81c163cfa0 --- /dev/null +++ b/packages/proxy/src/transfer-transport.test.ts @@ -0,0 +1,409 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import type { IncomingMessage } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterAll, expect, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { isTransferPhaseTimeoutError } from '@agent-device/host-kit/transport'; +import { + createNodeTransferFetch, + isTransferRoute, + settleUpstreamFetch, + transferWireDetails, +} from './transfer-transport.ts'; + +/** + * These cases run the real `node:http` client against real loopback servers, so they exercise the + * event order the transport depends on. Nothing here waits on a real 30- or 120-second ceiling: the + * phase budgets are covered at their owner in `@agent-device/host-kit` transport tests, and the + * budget arithmetic a transfer applies is asserted from the typed reason each failure carries. + */ + +const servers: http.Server[] = []; + +afterAll(async () => { + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); +}); + +/** Starts a loopback upstream and returns its origin. */ +function startUpstream(handler: http.RequestListener): Promise { + const server = http.createServer(handler); + servers.push(server); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${port}`); + }); + }); +} + +function transferFetch() { + return createNodeTransferFetch(); +} + +/** A streamed body that keeps producing chunks, so backpressure and drain waits are exercised. */ +function streamedBody(totalBytes: number, chunkBytes: number): ReadableStream { + let sent = 0; + return new ReadableStream({ + pull(controller) { + if (sent >= totalBytes) { + controller.close(); + return; + } + const size = Math.min(chunkBytes, totalBytes - sent); + sent += size; + controller.enqueue(new Uint8Array(size).fill(0x61)); + }, + }); +} + +test('the transfer route set covers the artifact routes and leaves the Fetch routes alone', () => { + expect( + [ + isTransferRoute('/upload', 'POST'), + isTransferRoute('/upload/preflight', 'POST'), + isTransferRoute('/upload/finalize', 'POST'), + isTransferRoute('/upload/direct/abc', 'PUT'), + isTransferRoute('/artifacts', 'GET'), + isTransferRoute('/artifacts/art-1', 'GET'), + ].every(Boolean), + ).toBe(true); + + expect( + [ + isTransferRoute('/rpc', 'POST'), + isTransferRoute('/health', 'GET'), + isTransferRoute('/sessions/s1/requests/r1/diagnostics', 'GET'), + isTransferRoute('/upload', 'GET'), + isTransferRoute('/upload/direct/abc', 'POST'), + isTransferRoute('/artifacts/art-1', 'POST'), + isTransferRoute('/upload/direct', 'PUT'), + ].some(Boolean), + ).toBe(false); +}); + +test('a location-less 308 chunk answer keeps its resume offsets', async () => { + const origin = await startUpstream((_req, res) => { + res.writeHead(308, { range: 'bytes=0-1023', 'x-upload-offset': '1024' }); + res.end(); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(1024)); + controller.close(); + }, + }), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(308); + expect(response.headers.get('x-upload-offset')).toBe('1024'); + expect(response.headers.get('range')).toBe('bytes=0-1023'); + expect(response.headers.get('location')).toBeNull(); + await response.text(); +}); + +test('a redirect with a Location is answered as a status, never followed', async () => { + const followed = await startUpstream((_req, res) => { + res.statusCode = 200; + res.end('followed'); + }); + const origin = await startUpstream((_req, res) => { + res.writeHead(307, { location: `${followed}/artifacts/art-1` }); + res.end(); + }); + + const response = await transferFetch()( + new Request(`${origin}/artifacts/art-1`, { method: 'GET' }), + ); + + expect(response.status).toBe(307); + expect(await response.text()).toBe(''); +}); + +test('an early 413 keeps its own status against a later peer break', async () => { + let upstreamRequest!: IncomingMessage; + const origin = await startUpstream((req, res) => { + upstreamRequest = req; + req.on('error', () => undefined); + res.statusCode = 413; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ ok: false, error: 'Artifact upload is too large' })); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: new ReadableStream({ + start(controller) { + for (let i = 0; i < 32; i += 1) controller.enqueue(new Uint8Array(64 * 1024)); + }, + }), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(413); + upstreamRequest.destroy(); + expect((await response.json()) as unknown).toEqual({ + ok: false, + error: 'Artifact upload is too large', + }); +}); + +test('an upload that is read completely answers with its status and streamed body', async () => { + const origin = await startUpstream(async (req, res) => { + let received = 0; + for await (const chunk of req) received += (chunk as Buffer).byteLength; + res.statusCode = 200; + res.end(`received:${received}`); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: streamedBody(256 * 1024, 16 * 1024), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(200); + expect(await response.text()).toBe('received:262144'); +}); + +test('a download body is handed over in order and ends exactly once', async () => { + const origin = await startUpstream((_req, res) => { + res.statusCode = 200; + res.setHeader('content-type', 'application/octet-stream'); + res.write(Buffer.alloc(1024, 0x62)); + res.write(Buffer.alloc(1024, 0x63)); + res.end(); + }); + + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + + expect(response.status).toBe(200); + const body = new Uint8Array(await response.arrayBuffer()); + expect(body).toHaveLength(2048); + expect(body[0]).toBe(0x62); + expect(body[1024]).toBe(0x63); +}); + +test('a null-body status and a HEAD answer carry headers with no body', async () => { + const origin = await startUpstream((req, res) => { + if (req.method === 'HEAD') { + res.writeHead(200, { 'content-length': '4096' }); + res.end(); + return; + } + res.writeHead(204, { 'x-artifact': 'gone' }); + res.end(); + }); + const fetchTransfer = transferFetch(); + + const head = await fetchTransfer(new Request(`${origin}/artifacts/art-1`, { method: 'HEAD' })); + const noContent = await fetchTransfer(new Request(`${origin}/artifacts/art-1`)); + + expect(head.status).toBe(200); + expect(head.headers.get('content-length')).toBe('4096'); + expect(head.body).toBeNull(); + expect(noContent.status).toBe(204); + expect(noContent.headers.get('x-artifact')).toBe('gone'); + expect(noContent.body).toBeNull(); +}); + +test('caller cancellation is reported with the caller reason verbatim', async () => { + const origin = await startUpstream(() => undefined); + const caller = new AbortController(); + const cancellation = new Error('caller went away'); + + const pending = transferFetch()( + new Request(`${origin}/artifacts/art-1`, { signal: caller.signal }), + ); + caller.abort(cancellation); + + await expect(pending).rejects.toBe(cancellation); +}); + +test('an upstream that dies before answering fails without a phase reason', async () => { + const origin = await startUpstream((req) => { + req.destroy(); + }); + + await expect(transferFetch()(new Request(`${origin}/artifacts/art-1`))).rejects.toBeInstanceOf( + Error, + ); +}); + +test('a forwarded request carries its method, path, query and allowed headers', async () => { + let seen: IncomingMessage | undefined; + const origin = await startUpstream((req, res) => { + seen = req; + res.statusCode = 200; + res.end('ok'); + }); + + const response = await transferFetch()( + new Request(`${origin}/artifacts/art-1?version=2`, { + headers: { 'x-artifact-hash': 'sha256:abc', authorization: 'Bearer daemon-secret' }, + }), + ); + await response.text(); + + expect(seen?.method).toBe('GET'); + expect(seen?.url).toBe('/artifacts/art-1?version=2'); + expect(seen?.headers['x-artifact-hash']).toBe('sha256:abc'); + expect(seen?.headers.authorization).toBe('Bearer daemon-secret'); +}); + +test('the default transport injects neither Fetch headers nor URL authorization', async () => { + let headers: http.IncomingHttpHeaders = {}; + const origin = await startUpstream((req, res) => { + headers = req.headers; + res.end('ok'); + }); + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + expect(await response.text()).toBe('ok'); + expect(headers.authorization).toBeUndefined(); + expect(headers.accept).toBeUndefined(); + expect(headers['user-agent']).toBeUndefined(); +}); + +test('a custom transport keeps its response body for the caller', async () => { + const response = await settleUpstreamFetch( + () => + Promise.resolve( + new Response('{"ok":true}', { + status: 201, + headers: { 'content-type': 'application/json' }, + }), + ), + new Request('http://daemon.internal:4310/upload/preflight', { method: 'POST' }), + ); + + expect(response.status).toBe(201); + expect(response.headers.get('content-type')).toBe('application/json'); + expect(await response.text()).toBe('{"ok":true}'); +}); + +test('the default transport serves an unchunked body whose length is known up front', async () => { + const payload = Buffer.alloc(4096, 0x7a); + const origin = await startUpstream((_req, res) => { + res.writeHead(200, { 'content-length': String(payload.byteLength) }); + res.end(payload); + }); + + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + + expect(response.status).toBe(200); + const body = new Uint8Array(await response.arrayBuffer()); + expect(body).toHaveLength(4096); + expect(body.every((byte) => byte === 0x7a)).toBe(true); +}); + +test('the forwarding aggregate ends a transport that never settles and names its phase', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const pending = settleUpstreamFetch( + () => new Promise(() => undefined), + new Request('http://daemon.internal:4310/artifacts/art-1'), + ); + let settled = false; + const failure = pending.catch((error: unknown) => error); + void failure.then(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(600_000 + 1); + expect(settled).toBe(true); + const error = await failure; + assert.ok(error instanceof AppError); + assert.equal(isTransferPhaseTimeoutError(error), false); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + } finally { + vi.useRealTimers(); + } +}); + +test('a transfer failure exposes the adopted wire details and nothing else does', () => { + const phaseFailure = new AppError( + 'COMMAND_FAILED', + 'Artifact transfer response-headers timed out', + { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }, + ); + + expect(transferWireDetails(phaseFailure)).toEqual({ + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }), + ), + ).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + expect( + transferWireDetails(new AppError('UNAUTHORIZED', 'Invalid proxy token', { reason: 'other' })), + ).toEqual({}); + expect(transferWireDetails(new Error('socket hang up'))).toEqual({}); +}); + +test('a transfer failure with unusable details stays off the wire', () => { + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: '', + timeoutMs: 30_000, + }), + ), + ).toEqual({}); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + timeoutMs: Number.NaN, + }), + ), + ).toEqual({}); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + }), + ), + ).toEqual({}); +}); diff --git a/packages/proxy/src/transfer-transport.ts b/packages/proxy/src/transfer-transport.ts new file mode 100644 index 0000000000..785b8dc0ad --- /dev/null +++ b/packages/proxy/src/transfer-transport.ts @@ -0,0 +1,481 @@ +import type { ClientRequest, IncomingMessage, RequestOptions } from 'node:http'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, + loadNodeHttpRequester, + observeClientRequestPhases, + observeCompletedRequestWrites, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + type TransferPhaseWatchdog, +} from '@agent-device/host-kit/transport'; + +const PROXY_TRANSFER_AGGREGATE_MS = 600_000; +const TRANSFER_FORWARDING_DEADLINE_REASON = 'transfer_forwarding_deadline_exceeded'; +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + +export function isTransferRoute(route: string, method: string): boolean { + if (route === '/upload' || route === '/upload/preflight' || route === '/upload/finalize') { + return method === 'POST'; + } + if (route.startsWith('/upload/direct/')) return method === 'PUT'; + return (route === '/artifacts' || route.startsWith('/artifacts/')) && method === 'GET'; +} + +/** Forwards one exchange over the runtime's default global Node agent, without redirects or retries. */ +export function createNodeTransferFetch(): (request: Request) => Promise { + return async (request) => { + const exchange = openExchange(request.signal); + try { + return await exchangeOverNodeHttp(request, exchange); + } catch (error) { + const failure = exchange.classify(error); + exchange.dispose(); + throw failure; + } + }; +} + +/** Bounds proxy-owned waits; opaque transport work and cancellation callbacks are never joined. */ +export async function settleUpstreamFetch( + upstreamFetch: (request: Request) => Promise, + request: Request, +): Promise { + const exchange = openExchange(request.signal); + let response: Response | undefined; + try { + response = await untilAborted( + exchange.signal, + () => upstreamFetch(new Request(request, { signal: exchange.signal })), + (late) => cancelForeignBody(late.body), + ); + if (!response.body) { + exchange.dispose(); + return response; + } + return new Response( + demandAwareStream(foreignBodySource(response.body), exchange, () => { + exchange.dispose(); + }), + { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }, + ); + } catch (error) { + const failure = exchange.classify(error); + cancelForeignBody(response?.body ?? null); + exchange.dispose(); + throw failure; + } +} + +export function transferWireDetails(error: unknown): Record { + if (!(error instanceof AppError) || error.code !== 'COMMAND_FAILED') return {}; + const { reason, transferPhase, timeoutMs } = error.details ?? {}; + if (reason === TRANSFER_FORWARDING_DEADLINE_REASON) { + return transferPhase === 'forwarding' && timeoutMs === PROXY_TRANSFER_AGGREGATE_MS + ? { reason, transferPhase, timeoutMs } + : {}; + } + if (!isTransferPhaseTimeoutError(error)) return {}; + let phaseBudgetMs: number; + switch (transferPhase) { + case 'connect': + phaseBudgetMs = TRANSFER_CONNECT_TIMEOUT_MS; + break; + case 'response-headers': + phaseBudgetMs = TRANSFER_RESPONSE_HEADER_TIMEOUT_MS; + break; + case 'request-body': + case 'response-body': + phaseBudgetMs = TRANSFER_BODY_INACTIVITY_TIMEOUT_MS; + break; + default: + return {}; + } + return timeoutMs === phaseBudgetMs ? { reason, transferPhase, timeoutMs } : {}; +} + +type TransferExchange = { + signal: AbortSignal; + watchdog: TransferPhaseWatchdog; + classify(error: unknown): unknown; + cancel(reason?: unknown): void; + dispose(): void; +}; + +function openExchange(caller: AbortSignal): TransferExchange { + const forwarding = new AbortController(); + const timer = setTimeout( + () => + forwarding.abort( + new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: TRANSFER_FORWARDING_DEADLINE_REASON, + transferPhase: 'forwarding', + timeoutMs: PROXY_TRANSFER_AGGREGATE_MS, + }), + ), + PROXY_TRANSFER_AGGREGATE_MS, + ); + timer.unref(); + const watchdog = createTransferPhaseWatchdog(AbortSignal.any([caller, forwarding.signal])); + return { + signal: watchdog.signal, + watchdog, + classify: (error) => (watchdog.signal.aborted ? watchdog.signal.reason : error), + cancel: (reason) => forwarding.abort(reason), + dispose: () => { + clearTimeout(timer); + watchdog.dispose(); + }, + }; +} + +async function exchangeOverNodeHttp( + request: Request, + exchange: TransferExchange, +): Promise { + const url = new URL(request.url); + const [transport, { urlToHttpOptions }] = await untilAborted(exchange.signal, () => + Promise.all([loadNodeHttpRequester(url.protocol), import('node:url')]), + ); + exchange.signal.throwIfAborted(); + const { auth: _auth, ...target } = urlToHttpOptions(url); + const options: RequestOptions = { + ...target, + method: request.method, + path: `${url.pathname}${url.search}`, + headers: Object.fromEntries(request.headers), + signal: exchange.signal, + }; + exchange.watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + return await new Promise((resolve, reject) => { + const outgoingStopped = new AbortController(); + const requestClosed = new AbortController(); + let answered = false; + let response: IncomingMessage | undefined; + let responseClosed: Promise = Promise.resolve(); + let finishing: Promise | undefined; + const finish = () => { + if (finishing) return finishing; + outgoingStopped.abort(); + // A flushed request can already have returned its socket to the global pool. + if (!req.writableFinished && !req.closed) req.destroy(); + finishing = Promise.all([pump, reqClosed, responseClosed]).then(() => { + phases.dispose(); + stopObservingWrites(); + req.off('error', fail); + response?.off('error', fail); + exchange.signal.removeEventListener('abort', onAbort); + exchange.dispose(); + }); + return finishing; + }; + const fail = (error: unknown) => { + if (answered) return; + answered = true; + const failure = exchange.classify(error); + response?.destroy(); + req.destroy(); + void finish().then(() => reject(failure)); + }; + const req = transport.request(options, (res) => { + response = res; + responseClosed = new Promise((closed) => res.once('close', closed)); + res.on('error', fail); + if (answered || exchange.signal.aborted) { + res.destroy(); + fail(exchange.signal.reason); + return; + } + const status = res.statusCode ?? 500; + if (status < 200 || status > 599) { + fail( + new AppError('COMMAND_FAILED', 'Upstream daemon returned an unusable HTTP status', { + reason: 'transfer_gateway_status', + status, + }), + ); + return; + } + answered = true; + outgoingStopped.abort(); + exchange.watchdog.suspend(); + const headers = new Headers(); + for (const [name, value] of Object.entries(res.headers)) { + if (value === undefined) continue; + for (const item of Array.isArray(value) ? value : [value]) headers.append(name, item); + } + if (request.method === 'HEAD' || NULL_BODY_STATUSES.has(status)) { + res.resume(); + void responseClosed.then(finish); + resolve(new Response(null, { status, headers })); + return; + } + resolve( + new Response(demandAwareStream(nodeBodySource(res), exchange, finish), { + status, + headers, + }), + ); + }); + const reqClosed = new Promise((closed) => { + req.once('close', () => { + requestClosed.abort(); + closed(); + if (!answered) + fail( + new AppError('COMMAND_FAILED', 'Upstream daemon closed before responding', { + reason: 'transfer_upstream_closed', + }), + ); + }); + }); + req.on('error', fail); + const phases = observeClientRequestPhases(req, exchange.watchdog); + const stopObservingWrites = observeCompletedRequestWrites(req, exchange.signal, () => { + exchange.watchdog.touchBody(); + }); + const onAbort = () => { + response?.destroy(); + fail(exchange.signal.reason); + }; + exchange.signal.addEventListener('abort', onAbort, { once: true }); + const pump = pumpOutgoing( + request, + req, + exchange.signal, + AbortSignal.any([exchange.signal, outgoingStopped.signal, requestClosed.signal]), + ).catch(fail); + if (exchange.signal.aborted) onAbort(); + }); +} + +async function pumpOutgoing( + request: Request, + req: ClientRequest, + lifetime: AbortSignal, + stopped: AbortSignal, +): Promise { + if (stopped.aborted) return; + if (!request.body) { + req.end(); + return; + } + const reader = request.body.getReader(); + try { + for (;;) { + const { done, value } = await untilAborted(stopped, () => reader.read()); + if (stopped.aborted) return; + if (done) { + req.end(); + return; + } + if (value.byteLength === 0) { + await yieldTransferEvents(stopped); + continue; + } + if (!req.write(value)) await waitUntilDrained(req, stopped); + } + } catch (error) { + if (!stopped.aborted) throw error; + } finally { + // Releasing a pending read rejects that read without cancelling the incoming client socket. + if (lifetime.aborted) void reader.cancel(lifetime.reason).catch(() => undefined); + reader.releaseLock(); + } +} + +function waitUntilDrained(req: ClientRequest, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + req.off('drain', drained); + signal.removeEventListener('abort', aborted); + }; + const drained = () => { + cleanup(); + resolve(); + }; + const aborted = () => { + cleanup(); + reject(signal.reason); + }; + req.once('drain', drained); + signal.addEventListener('abort', aborted, { once: true }); + if (signal.aborted) aborted(); + else if (req.destroyed || req.closed) { + cleanup(); + reject( + new AppError('COMMAND_FAILED', 'Upstream daemon closed during a write', { + reason: 'transfer_upstream_closed', + }), + ); + } else if (!req.writableNeedDrain) drained(); + }); +} + +type BodySource = { + next(): Promise; + stop(reason?: unknown): void | Promise; +}; + +function foreignBodySource(body: ReadableStream): BodySource { + const reader = body.getReader(); + return { + next: async () => { + const { done, value } = await reader.read(); + return done ? null : value; + }, + stop: (reason) => { + void reader.cancel(reason).catch(() => undefined); + reader.releaseLock(); + }, + }; +} + +function cancelForeignBody(body: ReadableStream | null): void { + if (body) void body.cancel().catch(() => undefined); +} + +function nodeBodySource(res: IncomingMessage): BodySource { + const iterator = res[Symbol.asyncIterator](); + return { + next: async () => { + const { done, value } = await iterator.next(); + return done ? null : value; + }, + stop: async () => { + if (!res.destroyed) res.destroy(); + await iterator.return?.(); + }, + }; +} + +function demandAwareStream( + source: BodySource, + exchange: TransferExchange, + cleanup: () => void | Promise, +): ReadableStream { + let ended = false; + let finishing: Promise | undefined; + const finish = (reason?: unknown) => { + if (finishing) return finishing; + ended = true; + exchange.signal.removeEventListener('abort', onAbort); + exchange.watchdog.suspend(); + finishing = Promise.resolve(source.stop(reason)) + .catch(() => undefined) + .then(cleanup); + return finishing; + }; + let downstream: ReadableStreamDefaultController; + const onAbort = () => { + if (ended) return; + downstream.error(exchange.signal.reason); + void finish(exchange.signal.reason); + }; + return new ReadableStream( + { + start(controller) { + downstream = controller; + exchange.signal.addEventListener('abort', onAbort, { once: true }); + if (exchange.signal.aborted) onAbort(); + }, + async pull(controller) { + if (ended) return; + exchange.watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + try { + for (;;) { + const chunk = await untilAborted(exchange.signal, () => source.next()); + if (ended) return; + if (chunk === null) { + await finish(); + controller.close(); + return; + } + if (chunk.byteLength === 0) { + await yieldTransferEvents(exchange.signal); + continue; + } + exchange.watchdog.suspend(); + controller.enqueue(chunk); + return; + } + } catch (error) { + if (ended) return; + const failure = exchange.classify(error); + controller.error(failure); + await finish(failure); + } + }, + cancel(reason) { + exchange.cancel(reason); + return finish(reason); + }, + }, + { highWaterMark: 0 }, + ); +} + +function yieldTransferEvents(signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener('abort', aborted); + resolve(); + }, 0); + const aborted = () => { + clearTimeout(timer); + signal.removeEventListener('abort', aborted); + reject(signal.reason); + }; + signal.addEventListener('abort', aborted, { once: true }); + if (signal.aborted) aborted(); + }); +} + +function untilAborted( + signal: AbortSignal, + run: () => Promise, + discardLate?: (value: T) => void, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const stop = () => { + settled = true; + signal.removeEventListener('abort', aborted); + }; + const aborted = () => { + stop(); + reject(signal.reason); + }; + if (signal.aborted) { + aborted(); + return; + } + signal.addEventListener('abort', aborted, { once: true }); + try { + run().then( + (value) => { + if (settled) { + discardLate?.(value); + return; + } + stop(); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + stop(); + reject(error); + }, + ); + } catch (error) { + stop(); + reject(error); + } + }); +} diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index bdabba0ca5..55e29a99a0 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -208,7 +208,7 @@ "packages/contracts/src/device-inventory.ts#readDevicePin": "sha256:bb15ca2a086a881c6cf945c25d432c092be165e58165d229c25df69c4c9fafbd", "packages/proxy/src/daemon-proxy.ts#rewriteUploadDirectUrl": "sha256:b121db3ffb5a520b4b31810a2e28ff01c1a15d959514c9fbe2a61cd960d2af93", "packages/proxy/src/daemon-proxy.ts#rewriteUploadPreflightResponse": "sha256:332820b6f10e079632330c2c4996c4faa6f339e921f12f287bd83c9296df5ccd", - "packages/proxy/src/daemon-proxy.ts#rewrittenUploadPreflightResponse": "sha256:21d633d947cd72f5008bcc886a8f3ecd4f778b594d0654b8d0741db8594003db", + "packages/proxy/src/daemon-proxy.ts#rewrittenUploadPreflightResponse": "sha256:cff01256a85981e8e581fcd1c11f9da70ed9811d3aae15eba0bec1eec489579c", "packages/daemon-contracts/src/daemon-resolution.ts#DAEMON_CAPABILITIES": "sha256:3478986d0e7b0e55e965c087a7b1dfd3cc018abdb372dba256cf90d27e63ed0b", "packages/daemon-contracts/src/daemon-resolution.ts#DaemonCapability": "sha256:2a5acf280a72e3d54c5ad7b52aece3ced37a30e4e721ef106a98e3bbd18ddbc3", "packages/daemon-contracts/src/daemon-resolution.ts#readDaemonCapabilities": "sha256:e410d36be74d307f7e5d8ddae63af5aae77f8eef77951ce9eeb7724f2fefae88", @@ -220,6 +220,11 @@ "packages/remote/src/session-script-destination.ts#RemoteEndpoint": "sha256:9c48e8e0ee14a25b85ebd930243fd263134836b35ecda55fb927919911263c15" }, "compatibleChanges": [ + { + "declaration": "packages/proxy/src/daemon-proxy.ts#rewrittenUploadPreflightResponse", + "digest": "sha256:cff01256a85981e8e581fcd1c11f9da70ed9811d3aae15eba0bec1eec489579c", + "rationale": "Preserves upstream 204, 205 and 304 as null-body preflight responses instead of failing ticket parsing or Response construction. Other successful statuses retain upload-ticket parsing and validation, and allowed headers are unchanged. Released protocol-2 upload clients treat 204/205 empty JSON as no preflight result and 304 as non-OK/redirect rejection, so they follow their existing fallback rather than misinterpreting an empty ticket. This is an observable repair of previously failing responses under the adopted finite forwarding policy, not an added optional field or a claim that all status outcomes are unchanged. Authentication, tenant confinement, upload credentials and ticket framing are unchanged." + }, { "declaration": "packages/kernel/src/contracts.ts#DaemonRequestMeta", "digest": "sha256:c2b14225388041f3a6e708c898c512f91b9b20e51a2656a1fa0d5bfa5989fa30", From 607b3bbc7fcd68b528f634f99b31bd7723c20ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 21:08:49 +0100 Subject: [PATCH 2/4] test(proxy): require reset evidence and clarify forwarding limits --- .../src/server/http/http-server-proxy-rollback.test.ts | 7 +++++-- packages/proxy/README.md | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts index e06e536d6f..f37811a93f 100644 --- a/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts +++ b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts @@ -8,6 +8,7 @@ import { closeLoopbackServer, createTestRuntimeStateHandle, listenOnLoopback, + skipWhenLoopbackUnavailable, } from '@agent-device/test-kit'; import { cleanupUploadedArtifact, @@ -25,9 +26,10 @@ function deferred() { return { promise, resolve, reject }; } -test.each(['disconnect and resume', 'uninterrupted'] as const)( +test.for(['disconnect and resume', 'uninterrupted'] as const)( 'an actual proxy-to-daemon upload handles %s and verifies finalized bytes', - async (mode) => { + async (mode, t) => { + if (await skipWhenLoopbackUnavailable(t, 'proxy-to-daemon upload coverage')) return; const runtime = createTestRuntimeStateHandle(); try { await runtime.run(async () => { @@ -128,6 +130,7 @@ test.each(['disconnect and resume', 'uninterrupted'] as const)( await rolledBack.promise; expect(fs.readFileSync(payload, 'utf8')).toBe('AB'); expect(client.writableEnded).toBe(false); + expect(errors.length).toBeGreaterThan(0); expect( errors.every((error) => (error as NodeJS.ErrnoException).code === 'ECONNRESET'), ).toBe(true); diff --git a/packages/proxy/README.md b/packages/proxy/README.md index f5042d9a8b..36f50d904c 100644 --- a/packages/proxy/README.md +++ b/packages/proxy/README.md @@ -111,7 +111,9 @@ Transfer phase and aggregate timeout error responses use HTTP 500 and include `r `reason: transfer_phase_timeout` and the matching connect/header (30000ms) or body (120000ms) phase budget, or `reason: transfer_forwarding_deadline_exceeded` with `transferPhase: forwarding` and `timeoutMs: 600000`. Other errors omit these fields. -A timeout while you consume an accepted response fails the body read instead of replacing its status. +After the proxy returns a response, a timeout while you consume it fails the body read instead of +replacing its status. Preflight bodies are consumed before ticket rewriting; a timeout there returns +HTTP 500. On these routes: @@ -126,7 +128,8 @@ Environment-proxy routing depends on your runtime. In loopback checks, Node 22.1 `HTTP_PROXY` and `HTTPS_PROXY` for both transports. Node 26.11.1 uses them when `NODE_USE_ENV_PROXY=1`. Bun 1.4.2 Fetch uses them without that opt-in, while the default transfer transport requires it. On Node 26.11.1 and Bun 1.4.2, `NO_PROXY=127.0.0.1` bypasses the proxy. -If you use Bun and rely on Fetch's proxy routing, set `NODE_USE_ENV_PROXY=1` or supply `upstreamFetch`. +If your Bun deployment needs proxy routing for transfers, set `NODE_USE_ENV_PROXY=1` or supply +`upstreamFetch`. By default, `/rpc`, `/health` and request diagnostics use global `fetch`. @@ -206,7 +209,7 @@ authenticate: async (request) => { | `authenticate` | none | Authenticates each client and names its tenant; replaces `clientToken`. | | `upstreamFetch` | none | Sends one request to the daemon. Without it, transfer routes use `node:http(s)` and every other route uses the global `fetch`. | | `maxRpcBodyBytes` | 64 MiB, the daemon's own request limit | Larger `/rpc` bodies get a 400 response. | -| `upstreamTimeoutMs` | none | Independent cap on one daemon request, including its body. Without it, a request lasts as long as the client waits, bounded by the client's command timeout and, on transfer routes, the 600-second forwarding aggregate. | +| `upstreamTimeoutMs` | none | Independent cap on one daemon request, including its body. `/health` keeps its separate five-second limit. Without this option, other requests last as long as the client waits, bounded by the client's command timeout and, on transfer routes, the 600-second forwarding aggregate. | `maxRpcBodyBytes` limits the bytes of one `/rpc` body. The proxy reads the whole body before it authenticates the client, then decodes and parses it. One unauthenticated request can therefore From 6873f3ee3f589d9c7c7e9e1543515f8778491955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:35:35 +0100 Subject: [PATCH 3/4] fix(proxy): centralize bounded transfer route ownership Keep transfer selection with proxy admission and load transfer mechanics only for transfer routes. Preserve the first classified failure while owned cleanup joins. Keep the independent foreign-preflight null-body repair outside this layer. --- .../proxy/src/daemon-proxy-transfer.test.ts | 153 ++++++++++++------ packages/proxy/src/daemon-proxy.ts | 29 ++-- .../src/transfer-transport-lifetime.test.ts | 47 ++++++ packages/proxy/src/transfer-transport.test.ts | 35 ---- packages/proxy/src/transfer-transport.ts | 11 +- test/wire-compat/ledger.json | 11 +- 6 files changed, 170 insertions(+), 116 deletions(-) diff --git a/packages/proxy/src/daemon-proxy-transfer.test.ts b/packages/proxy/src/daemon-proxy-transfer.test.ts index 13af18a45c..8ea02f1d9e 100644 --- a/packages/proxy/src/daemon-proxy-transfer.test.ts +++ b/packages/proxy/src/daemon-proxy-transfer.test.ts @@ -16,6 +16,13 @@ const options = { clientToken: 'client-secret', }; +const rpcBody = JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'agent-device.command', + params: { token: 'client-secret', command: 'version' }, +}); + async function listen(server: http.Server): Promise { server.listen(0, '127.0.0.1'); await once(server, 'listening'); @@ -255,15 +262,7 @@ test('a custom transport receives every route family including transfers', async headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, ...(method === 'POST' || method === 'PUT' ? { - body: - path === '/rpc' - ? JSON.stringify({ - jsonrpc: '2.0', - id: 1, - method: 'agent-device.command', - params: { token: 'client-secret', command: 'version' }, - }) - : '{}', + body: path === '/rpc' ? rpcBody : '{}', } : {}), }), @@ -286,6 +285,103 @@ test('a custom transport receives every route family including transfers', async ).toBe(true); }); +test('admitted transfers use Node while RPC and diagnostics use Fetch', async () => { + const daemon = await listen( + http.createServer((req, res) => { + req.resume(); + res.setHeader('content-type', 'application/json'); + res.end( + req.url === '/upload/preflight' + ? JSON.stringify({ + uploadId: 'ticket', + upload: { url: `http://${req.headers.host}/upload/direct/ticket` }, + }) + : '{"jsonrpc":"2.0","id":1,"result":{"ok":true}}', + ); + }), + ); + const node = vi.spyOn(http, 'request'); + const fetch = vi.spyOn(globalThis, 'fetch'); + onTestFinished(() => { + node.mockRestore(); + fetch.mockRestore(); + }); + const proxy = createDaemonProxy({ ...options, upstreamBaseUrl: daemon }); + for (const [method, path, transfer] of [ + ['POST', '/upload', true], + ['POST', '/upload/preflight', true], + ['POST', '/upload/finalize', true], + ['PUT', '/upload/direct/ticket', true], + ['GET', '/artifacts', true], + ['GET', '/artifacts/', true], + ['GET', '/artifacts/build', true], + ['POST', '/rpc', false], + ['GET', '/sessions/s/requests/r/diagnostics', false], + ] as const) { + node.mockClear(); + fetch.mockClear(); + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + ...(method === 'POST' || method === 'PUT' + ? { body: path === '/rpc' ? rpcBody : '{}' } + : {}), + }), + ); + expect(response.status, path).toBe(200); + await response.text(); + expect(node.mock.calls.length, path).toBe(transfer ? 1 : 0); + expect(fetch.mock.calls.length, path).toBe(transfer ? 0 : 1); + } + for (const [method, path] of [ + ['GET', '/upload'], + ['POST', '/upload/direct/x'], + ['POST', '/artifacts/x'], + ['PUT', '/upload/direct'], + ] as const) { + expect((await proxy.handle(new Request(`http://gateway.test${path}`, { method }))).status).toBe( + 404, + ); + } +}); + +test.each([false, true])( + 'non-transfer routes never load transfer mechanics (failure=%s)', + async (failure) => { + vi.resetModules(); + vi.doMock('./transfer-transport.ts', () => { + throw new Error('transfer mechanics loaded'); + }); + onTestFinished(() => { + vi.doUnmock('./transfer-transport.ts'); + vi.resetModules(); + }); + const { createDaemonProxy: create } = await import('./daemon-proxy.ts'); + const proxy = create({ + ...options, + upstreamFetch: async () => { + if (failure) throw new AppError('COMMAND_FAILED', 'original non-transfer failure'); + return Response.json({ jsonrpc: '2.0', id: 1, result: { ok: true } }); + }, + }); + for (const path of ['/rpc', '/health', '/sessions/s/requests/r/diagnostics']) { + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method: path === '/rpc' ? 'POST' : 'GET', + headers: { authorization: 'Bearer client-secret' }, + ...(path === '/rpc' ? { body: rpcBody } : {}), + }), + ); + expect(response.status, path).toBe(failure ? 500 : 200); + const body = await response.json(); + if (failure) expect(JSON.stringify(body)).toContain('original non-transfer failure'); + expect(body).not.toHaveProperty('transferPhase'); + expect(body).not.toHaveProperty('timeoutMs'); + } + }, +); + test('an explicit operator cap settles an ignored custom request and reaches its bounded signal', async () => { let bounded!: Request; const origin = await listen( @@ -367,42 +463,3 @@ test.each([205, 304, 400, 500, 599])( }); }, ); - -test.each([ - { transport: 'node', status: 204 }, - { transport: 'node', status: 205 }, - { transport: 'node', status: 304 }, - { transport: 'custom', status: 204 }, - { transport: 'custom', status: 205 }, - { transport: 'custom', status: 304 }, -])( - 'preflight preserves null-body $status and request identity through $transport transport', - async ({ transport, status }) => { - const headers = { 'x-request-id': 'daemon-preflight-request' }; - const daemon = await listen( - http.createServer((req, res) => { - req.resume(); - res.writeHead(status, headers); - res.end(); - }), - ); - const origin = await listen( - createDaemonProxyServer({ - ...options, - upstreamBaseUrl: daemon, - ...(transport === 'custom' - ? { upstreamFetch: async () => new Response(null, { status, headers }) } - : {}), - }), - ); - const response = await fetch(`${origin}/upload/preflight`, { - method: 'POST', - headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, - body: '{}', - redirect: 'error', - }); - expect(response.status).toBe(status); - expect(response.headers.get('x-request-id')).toBe('daemon-preflight-request'); - expect(await response.text()).toBe(''); - }, -); diff --git a/packages/proxy/src/daemon-proxy.ts b/packages/proxy/src/daemon-proxy.ts index 76c70182f5..8857d44a12 100644 --- a/packages/proxy/src/daemon-proxy.ts +++ b/packages/proxy/src/daemon-proxy.ts @@ -414,13 +414,6 @@ async function rewrittenUploadPreflightResponse(params: { publicOrigin: string | undefined; }): Promise { const { request, upstreamResponse, headers, uploadHeaders, publicOrigin } = params; - if ( - upstreamResponse.status === 204 || - upstreamResponse.status === 205 || - upstreamResponse.status === 304 - ) { - return new Response(null, { status: upstreamResponse.status, headers }); - } const text = await upstreamResponse.text(); headers.set('content-type', upstreamResponse.headers.get('content-type') ?? 'application/json'); return new Response( @@ -543,9 +536,9 @@ function normalizeProxyOptions(options: DaemonProxyOptions): NormalizedProxyOpti upstreamTimeoutMs: options.upstreamTimeoutMs, upstreamFetch, upstreamTransport: async (route, request) => { - const { isTransferRoute, createNodeTransferFetch, settleUpstreamFetch } = - await import('./transfer-transport.ts'); if (!isTransferRoute(route, request.method)) return upstreamFetch(request); + const { createNodeTransferFetch, settleUpstreamFetch } = + await import('./transfer-transport.ts'); return customUpstreamFetch ? settleUpstreamFetch(customUpstreamFetch, request) : createNodeTransferFetch()(request); @@ -584,9 +577,7 @@ function resolveProxyRoute(requestUrl: string): string { function isSupportedDaemonRoute(route: string, method: string): boolean { if (route === '/rpc') return method === 'POST'; - if (isSupportedUploadRoute(route, method)) return true; - if (route === '/artifacts' || route === '/artifacts/') return method === 'GET'; - if (route.startsWith('/artifacts/')) return method === 'GET'; + if (isTransferRoute(route, method)) return true; if (isRequestDiagnosticsRoute(route)) return method === 'GET'; return false; } @@ -608,6 +599,13 @@ function isRequestDiagnosticsRoute(route: string): boolean { ); } +function isTransferRoute(route: string, method: string): boolean { + return ( + isSupportedUploadRoute(route, method) || + (method === 'GET' && (route === '/artifacts' || route.startsWith('/artifacts/'))) + ); +} + function isSupportedUploadRoute(route: string, method: string): boolean { if (route === '/upload') return method === 'POST'; if (isUploadPreflightRoute(route)) return method === 'POST'; @@ -887,15 +885,16 @@ function rpcErrorResponse(rpcId: unknown, error: unknown): Response { } async function proxyErrorResponse(error: unknown, request: Request): Promise { - const { isTransferRoute, transferWireDetails } = await import('./transfer-transport.ts'); - const transfer = isTransferRoute(resolveProxyRoute(request.url), request.method); + const details = isTransferRoute(resolveProxyRoute(request.url), request.method) + ? (await import('./transfer-transport.ts')).transferWireDetails(error) + : {}; const normalized = normalizeError(error); return Response.json( { ok: false, error: normalized.message, code: normalized.code, - ...(transfer ? transferWireDetails(error) : {}), + ...details, }, { status: normalized.code === 'INVALID_ARGS' ? 400 : 500 }, ); diff --git a/packages/proxy/src/transfer-transport-lifetime.test.ts b/packages/proxy/src/transfer-transport-lifetime.test.ts index 815286ddc7..3622d8ef9d 100644 --- a/packages/proxy/src/transfer-transport-lifetime.test.ts +++ b/packages/proxy/src/transfer-transport-lifetime.test.ts @@ -801,3 +801,50 @@ test.each(['iterator stop', 'request close', 'outgoing pump'] as const)( expect(vi.getTimerCount()).toBe(0); }, ); + +test.each(['phase expiry', 'caller abort'] as const)( + 'a failure survives %s during joined cleanup', + async (late) => { + const entered = deferred(); + const origin = await upstream(() => entered.resolve()); + controlledClock(); + const closeEntered = deferred(); + const allowClose = deferred(); + onTestFinished(() => allowClose.resolve()); + const observed = observeRequest((req) => { + const once = req.once; + const registration = vi.spyOn(req, 'once').mockImplementation((event, listener) => { + if (event !== 'close') return Reflect.apply(once, req, [event, listener]); + registration.mockRestore(); + return Reflect.apply(once, req, [ + event, + (...args: unknown[]) => { + closeEntered.resolve(); + void allowClose.promise.then(() => Reflect.apply(listener, req, args)); + }, + ]); + }); + onTestFinished(() => registration.mockRestore()); + }); + const caller = new AbortController(); + const original = new AppError('COMMAND_FAILED', 'upstream failed before cleanup', { + reason: 'transfer_upstream_closed', + }); + let settled = false; + const result = createNodeTransferFetch()( + new Request(`${origin}/artifacts/build`, { signal: caller.signal }), + ).catch((error: unknown) => { + settled = true; + return error; + }); + await entered.promise; + (await observed.made).destroy(original); + await closeEntered.promise; + expect(settled).toBe(false); + if (late === 'phase expiry') await vi.advanceTimersByTimeAsync(30_000); + else caller.abort(new Error('later caller abort')); + allowClose.resolve(); + expect(await result).toBe(original); + expect(vi.getTimerCount()).toBe(0); + }, +); diff --git a/packages/proxy/src/transfer-transport.test.ts b/packages/proxy/src/transfer-transport.test.ts index 81c163cfa0..feff19b307 100644 --- a/packages/proxy/src/transfer-transport.test.ts +++ b/packages/proxy/src/transfer-transport.test.ts @@ -7,18 +7,10 @@ import { AppError } from '@agent-device/kernel/errors'; import { isTransferPhaseTimeoutError } from '@agent-device/host-kit/transport'; import { createNodeTransferFetch, - isTransferRoute, settleUpstreamFetch, transferWireDetails, } from './transfer-transport.ts'; -/** - * These cases run the real `node:http` client against real loopback servers, so they exercise the - * event order the transport depends on. Nothing here waits on a real 30- or 120-second ceiling: the - * phase budgets are covered at their owner in `@agent-device/host-kit` transport tests, and the - * budget arithmetic a transfer applies is asserted from the typed reason each failure carries. - */ - const servers: http.Server[] = []; afterAll(async () => { @@ -33,7 +25,6 @@ afterAll(async () => { ); }); -/** Starts a loopback upstream and returns its origin. */ function startUpstream(handler: http.RequestListener): Promise { const server = http.createServer(handler); servers.push(server); @@ -49,7 +40,6 @@ function transferFetch() { return createNodeTransferFetch(); } -/** A streamed body that keeps producing chunks, so backpressure and drain waits are exercised. */ function streamedBody(totalBytes: number, chunkBytes: number): ReadableStream { let sent = 0; return new ReadableStream({ @@ -65,31 +55,6 @@ function streamedBody(totalBytes: number, chunkBytes: number): ReadableStream { - expect( - [ - isTransferRoute('/upload', 'POST'), - isTransferRoute('/upload/preflight', 'POST'), - isTransferRoute('/upload/finalize', 'POST'), - isTransferRoute('/upload/direct/abc', 'PUT'), - isTransferRoute('/artifacts', 'GET'), - isTransferRoute('/artifacts/art-1', 'GET'), - ].every(Boolean), - ).toBe(true); - - expect( - [ - isTransferRoute('/rpc', 'POST'), - isTransferRoute('/health', 'GET'), - isTransferRoute('/sessions/s1/requests/r1/diagnostics', 'GET'), - isTransferRoute('/upload', 'GET'), - isTransferRoute('/upload/direct/abc', 'POST'), - isTransferRoute('/artifacts/art-1', 'POST'), - isTransferRoute('/upload/direct', 'PUT'), - ].some(Boolean), - ).toBe(false); -}); - test('a location-less 308 chunk answer keeps its resume offsets', async () => { const origin = await startUpstream((_req, res) => { res.writeHead(308, { range: 'bytes=0-1023', 'x-upload-offset': '1024' }); diff --git a/packages/proxy/src/transfer-transport.ts b/packages/proxy/src/transfer-transport.ts index 785b8dc0ad..e2849fa2c9 100644 --- a/packages/proxy/src/transfer-transport.ts +++ b/packages/proxy/src/transfer-transport.ts @@ -16,14 +16,6 @@ const PROXY_TRANSFER_AGGREGATE_MS = 600_000; const TRANSFER_FORWARDING_DEADLINE_REASON = 'transfer_forwarding_deadline_exceeded'; const NULL_BODY_STATUSES = new Set([204, 205, 304]); -export function isTransferRoute(route: string, method: string): boolean { - if (route === '/upload' || route === '/upload/preflight' || route === '/upload/finalize') { - return method === 'POST'; - } - if (route.startsWith('/upload/direct/')) return method === 'PUT'; - return (route === '/artifacts' || route.startsWith('/artifacts/')) && method === 'GET'; -} - /** Forwards one exchange over the runtime's default global Node agent, without redirects or retries. */ export function createNodeTransferFetch(): (request: Request) => Promise { return async (request) => { @@ -31,9 +23,8 @@ export function createNodeTransferFetch(): (request: Request) => Promise: prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." + "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers \u2014 sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added \u2014 an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." }, { "declaration": "packages/daemon/src/server/request-diagnostics-http.ts#RequestDiagnosticsHttpAuthorizer", From f7e0be4925b359639f5a9295d4824fb3cb367792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:41:35 +0100 Subject: [PATCH 4/4] test(proxy): assert the lazy-loader refusal premise Verify transfer mechanics cannot load in the lazy-boundary control and restore the wire ledger byte-for-byte to base after splitting the independent preflight repair. --- packages/proxy/src/daemon-proxy-transfer.test.ts | 3 +++ test/wire-compat/ledger.json | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/proxy/src/daemon-proxy-transfer.test.ts b/packages/proxy/src/daemon-proxy-transfer.test.ts index 8ea02f1d9e..54ef6aa5ac 100644 --- a/packages/proxy/src/daemon-proxy-transfer.test.ts +++ b/packages/proxy/src/daemon-proxy-transfer.test.ts @@ -365,6 +365,9 @@ test.each([false, true])( return Response.json({ jsonrpc: '2.0', id: 1, result: { ok: true } }); }, }); + const transfer = new Request('http://gateway.test/artifacts/x'); + transfer.headers.set('authorization', 'Bearer client-secret'); + await expect(proxy.handle(transfer)).rejects.toThrow(); for (const path of ['/rpc', '/health', '/sessions/s/requests/r/diagnostics']) { const response = await proxy.handle( new Request(`http://gateway.test${path}`, { diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index f50808face..bdabba0ca5 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -233,7 +233,7 @@ { "declaration": "packages/remote/src/daemon-artifacts.ts#DownloadRemoteArtifactParams", "digest": "sha256:3b0678250f72dca4ac1709103fd4c540dedd4d488eb3299ace5ce6b7c479676c", - "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state \u2014 never serialized, never sent to the daemon \u2014 that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved. #4511 adds the optional client-local privateFile field, passed through to the downloader. It is never sent, and every existing call site omits it." + "rationale": "#2246 adds the optional isDirectory field. It is pure client-local state — never serialized, never sent to the daemon — that tells the CLIENT'S OWN download logic to extract a tar body instead of writing it verbatim; the GET /artifacts/:id request and response framing are unchanged. Every existing call site (screenshot, recording) omits it and keeps writing a single file exactly as before. The signature also names the client request as DaemonWireRequest and the response as the kernel DaemonResponse instead of the daemon package aliases; only compile-time type names moved. #4511 adds the optional client-local privateFile field, passed through to the downloader. It is never sent, and every existing call site omits it." }, { "declaration": "packages/remote/src/daemon-artifacts.ts#downloadRemoteArtifact", @@ -308,7 +308,7 @@ { "declaration": "packages/daemon/src/server/http/http-authorization.ts#authorizeAuxiliaryHttpRequest", "digest": "sha256:1c323f97c3cefec8f95633b6abd127342bf75494bfb109d1cc6eadf3ceacaeae", - "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers \u2014 sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added \u2014 an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." + "rationale": "#2198 context: adds one optional field to the value this DAEMON-INTERNAL gate hands its own route handlers — sessionNamespace, the caller's tenant plus whether the daemon partitioned that caller's session names. Nothing about it is serialized: no request header is read that was not read before, no response field is added, and a refusal keeps the same 401 {ok:false,error,code} REST body sendRestJsonError already sent. What it enables is a refusal DROPPED, never one added — an unattested (client-declared) tenant no longer has the : prefix rule applied to it, because scopeRequestSession never applied that prefix to its sessions either. A released peer that now gets 200 where it got 401 is getting the ndjson record it asked for and already parses." }, { "declaration": "packages/daemon/src/server/request-diagnostics-http.ts#RequestDiagnosticsHttpAuthorizer",