diff --git a/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts new file mode 100644 index 0000000000..f37811a93f --- /dev/null +++ b/packages/daemon/src/server/http/http-server-proxy-rollback.test.ts @@ -0,0 +1,166 @@ +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { createDaemonProxyServer } from '@agent-device/proxy'; +import { + closeLoopbackServer, + createTestRuntimeStateHandle, + listenOnLoopback, + skipWhenLoopbackUnavailable, +} from '@agent-device/test-kit'; +import { + cleanupUploadedArtifact, + prepareUploadedArtifact, +} from '../../resources/artifact-tracking.ts'; +import { createDaemonHttpServer } from './http-server.ts'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +test.for(['disconnect and resume', 'uninterrupted'] as const)( + 'an actual proxy-to-daemon upload handles %s and verifies finalized bytes', + async (mode, t) => { + if (await skipWhenLoopbackUnavailable(t, 'proxy-to-daemon upload coverage')) return; + const runtime = createTestRuntimeStateHandle(); + try { + await runtime.run(async () => { + const daemon = await createDaemonHttpServer({ + token: 'daemon-secret', + handleRequest: async () => ({ ok: true, data: {} }), + env: process.env, + cwd: process.cwd(), + }); + const daemonPort = await listenOnLoopback(daemon); + const proxy = createDaemonProxyServer({ + upstreamBaseUrl: `http://127.0.0.1:${daemonPort}`, + upstreamToken: 'daemon-secret', + clientToken: 'client-secret', + }); + let tracked = ''; + try { + const origin = `http://127.0.0.1:${await listenOnLoopback(proxy)}`; + const content = Buffer.from('ABCDEF'); + const headers = { + authorization: 'Bearer client-secret', + 'content-type': 'application/json', + }; + const preflight = await fetch(`${origin}/upload/preflight`, { + method: 'POST', + headers, + body: JSON.stringify({ + uploadAttemptId: crypto.randomUUID(), + sha256: crypto.createHash('sha256').update(content).digest('hex'), + fileName: 'demo.apk', + sizeBytes: content.length, + artifactType: 'file', + platform: 'android', + }), + }); + expect(preflight.status).toBe(200); + const ticket = (await preflight.json()) as { + uploadId: string; + upload: { url: string; headers: Record }; + }; + expect(new URL(ticket.upload.url).origin).toBe(origin); + expect(ticket.upload.headers.authorization).toBe('Bearer client-secret'); + const put = (range: string, body: string) => + fetch(ticket.upload.url, { + method: 'PUT', + headers: { ...ticket.upload.headers, 'content-range': range }, + body, + }); + const first = await put('bytes 0-1/6', 'AB'); + expect(first.status).toBe(308); + expect(first.headers.get('x-upload-offset')).toBe('2'); + await first.text(); + if (mode === 'disconnect and resume') { + const prefix = deferred(); + const rolledBack = deferred(); + let payload = ''; + let watcher: fs.FSWatcher | undefined; + const createWriter = fs.createWriteStream; + const writerSpy = vi.spyOn(fs, 'createWriteStream').mockImplementation((...args) => { + const writer = Reflect.apply(createWriter, fs, args) as fs.WriteStream; + payload = String(args[0]); + writer.once('open', () => { + watcher = fs.watch(payload, () => { + void fs.promises.readFile(payload, 'utf8').then((bytes) => { + if (bytes === 'ABCD') prefix.resolve(bytes); + }, prefix.reject); + }); + }); + return writer; + }); + const truncate = fs.promises.truncate; + const rollbackSpy = vi + .spyOn(fs.promises, 'truncate') + .mockImplementation(async (path, length) => { + await truncate(path, length); + if (String(path) === payload && length === 2) rolledBack.resolve(); + }); + onTestFinished(() => { + watcher?.close(); + writerSpy.mockRestore(); + rollbackSpy.mockRestore(); + }); + const client = http.request(ticket.upload.url, { + method: 'PUT', + headers: { ...ticket.upload.headers, 'content-range': 'bytes 2-5/6' }, + }); + const errors: Error[] = []; + client.on('error', (error) => errors.push(error)); + onTestFinished(() => { + client.destroy(); + }); + client.write('CD'); + expect(await prefix.promise).toBe('ABCD'); + assert.equal(fs.readFileSync(payload, 'utf8'), 'ABCD'); + const closed = new Promise((resolve) => client.once('close', () => resolve())); + client.destroy(); + await closed; + await rolledBack.promise; + expect(fs.readFileSync(payload, 'utf8')).toBe('AB'); + expect(client.writableEnded).toBe(false); + expect(errors.length).toBeGreaterThan(0); + expect( + errors.every((error) => (error as NodeJS.ErrnoException).code === 'ECONNRESET'), + ).toBe(true); + watcher?.close(); + writerSpy.mockRestore(); + rollbackSpy.mockRestore(); + const probe = await put('bytes 0-1/6', 'AB'); + expect(probe.status).toBe(308); + expect(probe.headers.get('x-upload-offset')).toBe('2'); + await probe.text(); + } + const complete = await put('bytes 2-5/6', 'CDEF'); + expect(complete.status).toBe(200); + await complete.text(); + const finalize = await fetch(`${origin}/upload/finalize`, { + method: 'POST', + headers, + body: JSON.stringify({ uploadId: ticket.uploadId }), + }); + expect(finalize.status).toBe(200); + tracked = ((await finalize.json()) as { uploadId: string }).uploadId; + expect(fs.readFileSync(prepareUploadedArtifact(tracked))).toEqual(content); + } finally { + if (tracked) cleanupUploadedArtifact(tracked); + await closeLoopbackServer(proxy); + await closeLoopbackServer(daemon); + } + }); + } finally { + runtime.dispose(); + } + }, +); diff --git a/packages/host-kit/src/internal/transport-phases.test.ts b/packages/host-kit/src/internal/transport-phases.test.ts index 68865afe46..fafe9784cf 100644 --- a/packages/host-kit/src/internal/transport-phases.test.ts +++ b/packages/host-kit/src/internal/transport-phases.test.ts @@ -207,6 +207,38 @@ test('disposing the phase observer removes only its own listeners', () => { assert.equal(fake.listenerCount('response'), 0); }); +test('suspend resumes counting when the same phase is armed again', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 10_000); + watchdog.suspend(); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 3); + assert.equal(watchdog.signal.aborted, false); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1); + assertPhaseDetails(watchdog.signal.reason, 'response-body', 120_000); +}); + +test('body progress during a suspend cannot restart the stopped ceiling', async () => { + const watchdog = watchPhases(); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + watchdog.suspend(); + watchdog.touchBody(); + await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 2); + assert.equal(watchdog.signal.aborted, false); +}); + +test('caller cancellation still binds an exchange whose idle ceiling is suspended', async () => { + const caller = new AbortController(); + const watchdog = watchPhases(caller.signal); + watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + watchdog.suspend(); + const cancellation = new Error('caller went away'); + caller.abort(cancellation); + assert.equal(watchdog.signal.aborted, true); + assert.equal(watchdog.signal.reason, cancellation); +}); + function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void { assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`); assert.equal(isTransferPhaseTimeoutError(reason), true); diff --git a/packages/host-kit/src/internal/transport.ts b/packages/host-kit/src/internal/transport.ts index d698985b1e..165236b05e 100644 --- a/packages/host-kit/src/internal/transport.ts +++ b/packages/host-kit/src/internal/transport.ts @@ -160,6 +160,9 @@ export type TransferPhaseWatchdog = { signal: AbortSignal; arm(phase: TransferPhase, timeoutMs: number): void; touchBody(): void; + /** Stops counting an idle ceiling while the holder waits on demand, keeping its armed phase. */ + suspend(): void; + /** Stops observing this attempt. A suspended phase stays suspended. */ dispose(): void; }; @@ -168,11 +171,12 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal; let armed: { phase: TransferPhase; timeoutMs: number } | undefined; let timer: ReturnType | undefined; + let suspended = false; const rearm = () => { if (timer !== undefined) clearTimeout(timer); timer = undefined; const current = armed; - if (!current) return; + if (!current || suspended) return; timer = setTimeout(() => { timer = undefined; phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs)); @@ -183,6 +187,7 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe signal, arm(nextPhase, timeoutMs) { if (signal.aborted) return; + suspended = false; armed = { phase: nextPhase, timeoutMs }; rearm(); }, @@ -190,6 +195,11 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe if (!armed || !BODY_PHASES.has(armed.phase)) return; rearm(); }, + suspend() { + suspended = true; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }, dispose() { armed = undefined; if (timer !== undefined) clearTimeout(timer); diff --git a/packages/host-kit/src/transport.ts b/packages/host-kit/src/transport.ts index 16202f2ff3..eefbc20b97 100644 --- a/packages/host-kit/src/transport.ts +++ b/packages/host-kit/src/transport.ts @@ -8,6 +8,9 @@ export { readNodeHttpRequestBody, readNodeHttpResponseBody, timingSafeStringEqual, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + type TransferPhaseWatchdog, type NodeHttpRequester, } from './internal/transport.ts'; diff --git a/packages/proxy/README.md b/packages/proxy/README.md index 94b28e743c..36f50d904c 100644 --- a/packages/proxy/README.md +++ b/packages/proxy/README.md @@ -94,10 +94,49 @@ Your Node HTTP client must send exactly one valid `Host` header on every request configured. Configure raw-socket and HTTP/1.0 health probes to send `Host: localhost` (or your deployment's host and port). +### How the proxy reaches the daemon + +On artifact transfer routes — `POST /upload`, `/upload/preflight` and `/upload/finalize`, +`PUT /upload/direct/*` and `GET /artifacts*` — the proxy forwards with `node:http` / `node:https`. +Each transfer has a connect limit of 30 seconds, a response-header wait of 30 seconds that +starts only once its own outgoing body has finished and the connection is established, and a +120-second limit on body inactivity. For downloads, that limit counts time spent waiting for +upstream bytes while you read the response; it pauses when you stop reading. Every forwarded +exchange also ends at 600 seconds regardless of how steadily its bytes move. Your `upstreamTimeoutMs` +is an additional cap. The earliest caller cancellation, operator cap, phase limit or aggregate expiry +ends the exchange. A transfer does not retry or restart its aggregate. + +Transfer phase and aggregate timeout error responses use HTTP 500 and include `reason`, +`transferPhase` and `timeoutMs`. These fields appear only for `COMMAND_FAILED` with +`reason: transfer_phase_timeout` and the matching connect/header (30000ms) or body (120000ms) +phase budget, or `reason: transfer_forwarding_deadline_exceeded` with +`transferPhase: forwarding` and `timeoutMs: 600000`. Other errors omit these fields. +After the proxy returns a response, a timeout while you consume it fails the body read instead of +replacing its status. Preflight bodies are consumed before ticket rewriting; a timeout there returns +HTTP 500. + +On these routes: + +- Redirects are returned without being followed. The daemon's `308` resume response keeps its + `Range` and `x-upload-offset` headers. +- Default request headers added by `fetch`, such as `accept` and `user-agent`, are not sent. +- A patched global `fetch` or an `undici` global dispatcher no longer intercepts the request. + +Supply `upstreamFetch` if you need those Fetch behaviors or your own routing. + +Environment-proxy routing depends on your runtime. In loopback checks, Node 22.12 ignores +`HTTP_PROXY` and `HTTPS_PROXY` for both transports. Node 26.11.1 uses them when +`NODE_USE_ENV_PROXY=1`. Bun 1.4.2 Fetch uses them without that opt-in, while the default transfer +transport requires it. On Node 26.11.1 and Bun 1.4.2, `NO_PROXY=127.0.0.1` bypasses the proxy. +If your Bun deployment needs proxy routing for transfers, set `NODE_USE_ENV_PROXY=1` or supply +`upstreamFetch`. + +By default, `/rpc`, `/health` and request diagnostics use global `fetch`. + ### Reach the daemon over your own transport -By default the proxy calls the daemon with the global `fetch`. Pass `upstreamFetch` when the daemon -is only reachable some other way, such as a tunnel the device host opened to your gateway: +Pass `upstreamFetch` when the daemon is only reachable some other way, such as a tunnel the device +host opened to your gateway: ```ts const proxy = createDaemonProxy({ @@ -111,6 +150,16 @@ const proxy = createDaemonProxy({ `upstreamFetch` receives a `Request` addressed to `upstreamBaseUrl` and must resolve with the daemon's `Response`. Abort the exchange when `request.signal` aborts. +On transfer routes, the proxy bounds its wait for your transport and response body. It settles on +your abort, your `upstreamTimeoutMs`, its 600-second aggregate and its demand-aware response-body +inactivity limit. Work your transport starts can keep running if it ignores `request.signal`. The +proxy does not wait for that work or for a cancellation callback that never settles. It cannot +measure a connect or a header wait it does not own, so on transfer routes your transport keeps those +obligations. Bound connection establishment to 30 seconds and the response-header wait to 30 seconds +once the connection is established and the outgoing body has finished. Stream `request.body` with +120-second body inactivity limits. Stop your request and response sources on cancellation. A body +pull alone does not confirm that an outgoing byte was sent. + ### Authenticate each client yourself One `clientToken` is shared by every client. To give each client its own credential, pass @@ -158,9 +207,9 @@ authenticate: async (request) => { | `upstreamToken` | required | The daemon's auth token. | | `clientToken` | required without `authenticate` | Token clients must send as their daemon auth token. | | `authenticate` | none | Authenticates each client and names its tenant; replaces `clientToken`. | -| `upstreamFetch` | global `fetch` | Sends one request to the daemon. | +| `upstreamFetch` | none | Sends one request to the daemon. Without it, transfer routes use `node:http(s)` and every other route uses the global `fetch`. | | `maxRpcBodyBytes` | 64 MiB, the daemon's own request limit | Larger `/rpc` bodies get a 400 response. | -| `upstreamTimeoutMs` | none | Longest time one daemon request may take, including its body. Without it, a request lasts as long as the client waits, and the client's command timeout bounds it. | +| `upstreamTimeoutMs` | none | Independent cap on one daemon request, including its body. `/health` keeps its separate five-second limit. Without this option, other requests last as long as the client waits, bounded by the client's command timeout and, on transfer routes, the 600-second forwarding aggregate. | `maxRpcBodyBytes` limits the bytes of one `/rpc` body. The proxy reads the whole body before it authenticates the client, then decodes and parses it. One unauthenticated request can therefore diff --git a/packages/proxy/src/daemon-proxy-transfer.test.ts b/packages/proxy/src/daemon-proxy-transfer.test.ts new file mode 100644 index 0000000000..54ef6aa5ac --- /dev/null +++ b/packages/proxy/src/daemon-proxy-transfer.test.ts @@ -0,0 +1,468 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createDaemonProxy, + createDaemonProxyServer, + type DaemonProxyOptions, +} from './daemon-proxy.ts'; + +const options = { + upstreamBaseUrl: 'http://daemon.internal:4310', + upstreamToken: 'daemon-secret', + clientToken: 'client-secret', +}; + +const rpcBody = JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'agent-device.command', + params: { token: 'client-secret', command: 'version' }, +}); + +async function listen(server: http.Server): Promise { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function requestThroughProxy(origin: string, path: string, method = 'POST') { + return new Promise<{ status: number; body: Record }>((resolve, reject) => { + const req = http.request( + `${origin}${path}`, + { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + }, + (res) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + text += chunk; + }); + res.on('error', reject); + res.on('end', () => + resolve({ status: res.statusCode!, body: text ? JSON.parse(text) : {} }), + ); + }, + ); + req.on('error', reject); + req.end(method === 'POST' ? '{}' : undefined); + }); +} + +test('the actual proxy HTTP response exposes a header timeout as 500 with exactly three transfer details', async () => { + let entered!: () => void; + const uploaded = new Promise((resolve) => { + entered = resolve; + }); + const daemon = await listen( + http.createServer((req) => { + req.resume(); + req.once('end', entered); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const original = http.request; + let flushed!: (request: http.ClientRequest) => void; + const finished = new Promise((resolve) => { + flushed = resolve; + }); + const observer = vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(original, http, args) as http.ClientRequest; + if (req.getHeader('authorization') === 'Bearer daemon-secret') { + req.once('finish', () => flushed(req)); + } + return req; + }); + onTestFinished(() => observer.mockRestore()); + const response = requestThroughProxy(origin, '/upload/preflight'); + await uploaded; + expect((await finished).writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(30_000); + expect(await response).toEqual({ + status: 500, + body: { + ok: false, + error: 'Artifact transfer response-headers timed out', + code: 'COMMAND_FAILED', + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }, + }); +}); + +test('the actual HTTP response exposes the forwarding aggregate when a custom transport ignores cancellation', async () => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamFetch: () => { + entered(); + return new Promise(() => undefined); + }, + }), + ); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const response = requestThroughProxy(origin, '/upload/preflight'); + await started; + await vi.advanceTimersByTimeAsync(600_000); + expect(await response).toEqual({ + status: 500, + body: { + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }, + }); +}); + +test.each([ + { + code: 'INVALID_ARGS', + status: 400, + phase: 'response-body', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'unknown-phase', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'response-headers', + budget: 120_000, + reason: 'transfer_phase_timeout', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'forwarding', + budget: 600_001, + reason: 'transfer_forwarding_deadline_exceeded', + }, + { + code: 'COMMAND_FAILED', + status: 500, + phase: 'response-body', + budget: 600_000, + reason: 'transfer_forwarding_deadline_exceeded', + }, +] as const)( + 'actual HTTP $status omits invalid transfer detail combinations ($code/$phase/$budget)', + async ({ code, status, phase, budget, reason }) => { + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamFetch: async () => { + throw new AppError(code, 'Invalid transfer metadata', { + reason, + transferPhase: phase, + timeoutMs: budget, + secret: 'hidden', + }); + }, + }), + ); + expect(await requestThroughProxy(origin, '/upload/preflight')).toEqual({ + status, + body: { ok: false, error: 'Invalid transfer metadata', code }, + }); + }, +); + +test('recognized forwarding details are additive on transfers and absent on diagnostics and health', async () => { + const proxy = createDaemonProxy({ + ...options, + upstreamFetch: async () => { + throw new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + secret: 'hidden', + }); + }, + }); + const send = (path: string) => + proxy.handle( + new Request(`http://gateway.test${path}`, { + headers: { authorization: 'Bearer client-secret' }, + }), + ); + expect(await (await send('/artifacts/build')).json()).toEqual({ + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + for (const path of ['/health', '/sessions/session/requests/request/diagnostics']) { + expect(await (await send(path)).json()).toEqual({ + ok: false, + error: 'Proxy transfer forwarding deadline exceeded', + code: 'COMMAND_FAILED', + }); + } +}); + +test('a custom transport receives every route family including transfers', async () => { + const seen: Request[] = []; + const custom: DaemonProxyOptions['upstreamFetch'] = async (request) => { + seen.push(request); + const path = new URL(request.url).pathname; + if (path === '/upload/preflight') + return Response.json({ + uploadId: 'ticket', + upload: { url: 'http://daemon.internal:4310/upload/direct/ticket' }, + }); + if (path === '/rpc') return Response.json({ jsonrpc: '2.0', id: 1, result: { ok: true } }); + return new Response('custom answer'); + }; + const proxy = createDaemonProxy({ ...options, upstreamFetch: custom }); + for (const [method, path] of [ + ['POST', '/upload'], + ['POST', '/upload/preflight'], + ['POST', '/upload/finalize'], + ['PUT', '/upload/direct/ticket'], + ['GET', '/artifacts/build'], + ['GET', '/sessions/session/requests/request/diagnostics'], + ['GET', '/health'], + ['POST', '/rpc'], + ]) { + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + ...(method === 'POST' || method === 'PUT' + ? { + body: path === '/rpc' ? rpcBody : '{}', + } + : {}), + }), + ); + expect(response.status, path).toBe(200); + await response.text(); + } + expect(seen.map((request) => new URL(request.url).pathname)).toEqual([ + '/upload', + '/upload/preflight', + '/upload/finalize', + '/upload/direct/ticket', + '/artifacts/build', + '/sessions/session/requests/request/diagnostics', + '/health', + '/rpc', + ]); + expect( + seen.every((request) => request.headers.get('authorization') === 'Bearer daemon-secret'), + ).toBe(true); +}); + +test('admitted transfers use Node while RPC and diagnostics use Fetch', async () => { + const daemon = await listen( + http.createServer((req, res) => { + req.resume(); + res.setHeader('content-type', 'application/json'); + res.end( + req.url === '/upload/preflight' + ? JSON.stringify({ + uploadId: 'ticket', + upload: { url: `http://${req.headers.host}/upload/direct/ticket` }, + }) + : '{"jsonrpc":"2.0","id":1,"result":{"ok":true}}', + ); + }), + ); + const node = vi.spyOn(http, 'request'); + const fetch = vi.spyOn(globalThis, 'fetch'); + onTestFinished(() => { + node.mockRestore(); + fetch.mockRestore(); + }); + const proxy = createDaemonProxy({ ...options, upstreamBaseUrl: daemon }); + for (const [method, path, transfer] of [ + ['POST', '/upload', true], + ['POST', '/upload/preflight', true], + ['POST', '/upload/finalize', true], + ['PUT', '/upload/direct/ticket', true], + ['GET', '/artifacts', true], + ['GET', '/artifacts/', true], + ['GET', '/artifacts/build', true], + ['POST', '/rpc', false], + ['GET', '/sessions/s/requests/r/diagnostics', false], + ] as const) { + node.mockClear(); + fetch.mockClear(); + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method, + headers: { authorization: 'Bearer client-secret', 'content-type': 'application/json' }, + ...(method === 'POST' || method === 'PUT' + ? { body: path === '/rpc' ? rpcBody : '{}' } + : {}), + }), + ); + expect(response.status, path).toBe(200); + await response.text(); + expect(node.mock.calls.length, path).toBe(transfer ? 1 : 0); + expect(fetch.mock.calls.length, path).toBe(transfer ? 0 : 1); + } + for (const [method, path] of [ + ['GET', '/upload'], + ['POST', '/upload/direct/x'], + ['POST', '/artifacts/x'], + ['PUT', '/upload/direct'], + ] as const) { + expect((await proxy.handle(new Request(`http://gateway.test${path}`, { method }))).status).toBe( + 404, + ); + } +}); + +test.each([false, true])( + 'non-transfer routes never load transfer mechanics (failure=%s)', + async (failure) => { + vi.resetModules(); + vi.doMock('./transfer-transport.ts', () => { + throw new Error('transfer mechanics loaded'); + }); + onTestFinished(() => { + vi.doUnmock('./transfer-transport.ts'); + vi.resetModules(); + }); + const { createDaemonProxy: create } = await import('./daemon-proxy.ts'); + const proxy = create({ + ...options, + upstreamFetch: async () => { + if (failure) throw new AppError('COMMAND_FAILED', 'original non-transfer failure'); + return Response.json({ jsonrpc: '2.0', id: 1, result: { ok: true } }); + }, + }); + const transfer = new Request('http://gateway.test/artifacts/x'); + transfer.headers.set('authorization', 'Bearer client-secret'); + await expect(proxy.handle(transfer)).rejects.toThrow(); + for (const path of ['/rpc', '/health', '/sessions/s/requests/r/diagnostics']) { + const response = await proxy.handle( + new Request(`http://gateway.test${path}`, { + method: path === '/rpc' ? 'POST' : 'GET', + headers: { authorization: 'Bearer client-secret' }, + ...(path === '/rpc' ? { body: rpcBody } : {}), + }), + ); + expect(response.status, path).toBe(failure ? 500 : 200); + const body = await response.json(); + if (failure) expect(JSON.stringify(body)).toContain('original non-transfer failure'); + expect(body).not.toHaveProperty('transferPhase'); + expect(body).not.toHaveProperty('timeoutMs'); + } + }, +); + +test('an explicit operator cap settles an ignored custom request and reaches its bounded signal', async () => { + let bounded!: Request; + const origin = await listen( + createDaemonProxyServer({ + ...options, + upstreamTimeoutMs: 10, + upstreamFetch: (request) => { + bounded = request; + return new Promise(() => undefined); + }, + }), + ); + const response = await requestThroughProxy(origin, '/upload/preflight'); + expect(response.status).toBe(500); + assert.ok(bounded.signal.aborted); + expect(bounded.signal.reason.name).toBe('TimeoutError'); + expect(response.body).not.toHaveProperty('transferPhase'); + expect(response.body).not.toHaveProperty('timeoutMs'); +}); + +test.each([401, 413, 200])( + 'an early %s crosses the real proxy HTTP listener while ingress remains unfinished', + async (status) => { + const daemon = await listen( + http.createServer((_req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end('{"answer":"intact"}'); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + const response = await new Promise<{ status: number; text: string }>((resolve, reject) => { + const req = http.request( + `${origin}/upload/direct/ticket`, + { + method: 'PUT', + headers: { + authorization: 'Bearer client-secret', + 'content-type': 'application/octet-stream', + }, + }, + (res) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => { + text += chunk; + }); + res.once('error', reject); + res.once('end', () => { + expect(req.writableEnded).toBe(false); + resolve({ status: res.statusCode!, text }); + req.destroy(); + }); + }, + ); + req.on('error', reject); + req.write(Buffer.from('unfinished ingress')); + onTestFinished(() => { + req.destroy(); + }); + }); + expect(response).toEqual({ status, text: '{"answer":"intact"}' }); + }, +); + +test.each([205, 304, 400, 500, 599])( + 'a final upstream %s crosses the real HTTP proxy without normalization', + async (status) => { + const noBody = status === 205 || status === 304; + const daemon = await listen( + http.createServer((_req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(noBody ? undefined : '{"answer":"upstream"}'); + }), + ); + const origin = await listen(createDaemonProxyServer({ ...options, upstreamBaseUrl: daemon })); + expect(await requestThroughProxy(origin, '/artifacts/build', 'GET')).toEqual({ + status, + body: noBody ? {} : { answer: 'upstream' }, + }); + }, +); diff --git a/packages/proxy/src/daemon-proxy.ts b/packages/proxy/src/daemon-proxy.ts index 38c6c897ae..8857d44a12 100644 --- a/packages/proxy/src/daemon-proxy.ts +++ b/packages/proxy/src/daemon-proxy.ts @@ -25,9 +25,14 @@ import { } from './install-source-admission.ts'; /** - * Carries one request to the upstream daemon and resolves with its response. The default is the - * global `fetch`; supply your own to reach the daemon over another transport, such as a - * WebSocket tunnel. Abort the exchange when `request.signal` aborts. + * Carries one request to the upstream daemon and resolves with its response. Without `upstreamFetch` + * the proxy reaches the daemon over `node:http(s)` on artifact transfer routes and over the global + * `fetch` on `/rpc`, health and diagnostics. Supply your own to reach the daemon over another + * transport, such as a WebSocket tunnel. Abort the exchange when `request.signal` aborts. + * + * A custom transport owns its phase guarantees and must observe `request.signal` on every route. + * On transfer routes, the proxy bounds its own response and body waits, even if the transport + * ignores cancellation. It cannot forcibly join work started by that transport. */ export type DaemonProxyUpstreamFetch = (request: Request) => Promise; @@ -60,10 +65,12 @@ export type DaemonProxyOptions = { publicOrigin?: string; maxRpcBodyBytes?: number; /** - * Longest time one daemon request may take, including its body. Unset, an exchange lasts as long - * as its client request: the client's command envelope bounds it. + * Independent cap on one upstream request, including its response body. Transfers also observe + * their phase limits and 600-second aggregate. When unset, other requests use the client's + * command envelope. The health probe keeps its separate five-second limit. */ upstreamTimeoutMs?: number; + /** Overrides the transport for every upstream route. See `DaemonProxyUpstreamFetch`. */ upstreamFetch?: DaemonProxyUpstreamFetch; } & ( | { @@ -102,6 +109,7 @@ type NormalizedProxyOptions = { maxRpcBodyBytes: number; upstreamTimeoutMs: number | undefined; upstreamFetch: DaemonProxyUpstreamFetch; + upstreamTransport: (route: string, request: Request) => Promise; }; /** The client a request was authenticated as; `tenantId` is set only by `authenticate`. */ @@ -141,7 +149,7 @@ export function createDaemonProxy(options: DaemonProxyOptions): DaemonProxy { try { return await handleProxyRequest(request, normalized, instanceId); } catch (error) { - return proxyErrorResponse(error); + return proxyErrorResponse(error, request); } }, }; @@ -349,7 +357,8 @@ async function forwardProxyRequest(params: { }): Promise { const { request, route, options, rpcBody, caller } = params; const body = resolveUpstreamBody(request, route, rpcBody, options.upstreamToken, caller.tenantId); - const upstreamResponse = await options.upstreamFetch( + const upstreamResponse = await options.upstreamTransport( + route, new Request(buildUpstreamUrl(options.upstreamBaseUrl, route, request.url), { method: request.method, headers: buildUpstreamHeaders(request.headers, options.upstreamToken, route, caller.tenantId), @@ -371,10 +380,6 @@ async function forwardProxyRequest(params: { return new Response(upstreamResponse.body, { status: upstreamResponse.status, headers }); } -/** - * An upstream exchange lives exactly as long as the client keeps waiting for it: the daemon - * turns a dropped upstream request into cancellation of in-flight runner work. - */ function upstreamSignal(request: Request, options: NormalizedProxyOptions): AbortSignal { if (options.upstreamTimeoutMs === undefined) return request.signal; return AbortSignal.any([request.signal, AbortSignal.timeout(options.upstreamTimeoutMs)]); @@ -516,6 +521,8 @@ function normalizeProxyOptions(options: DaemonProxyOptions): NormalizedProxyOpti const clientToken = options.authenticate ? undefined : normalizeToken(options.clientToken ?? '', 'clientToken'); + const upstreamFetch = options.upstreamFetch ?? ((request) => fetch(request)); + const customUpstreamFetch = options.upstreamFetch; return { upstreamBaseUrl, publicOrigin: @@ -527,7 +534,15 @@ function normalizeProxyOptions(options: DaemonProxyOptions): NormalizedProxyOpti authenticate: options.authenticate, maxRpcBodyBytes: options.maxRpcBodyBytes ?? DEFAULT_MAX_RPC_BODY_BYTES, upstreamTimeoutMs: options.upstreamTimeoutMs, - upstreamFetch: options.upstreamFetch ?? ((request) => fetch(request)), + upstreamFetch, + upstreamTransport: async (route, request) => { + if (!isTransferRoute(route, request.method)) return upstreamFetch(request); + const { createNodeTransferFetch, settleUpstreamFetch } = + await import('./transfer-transport.ts'); + return customUpstreamFetch + ? settleUpstreamFetch(customUpstreamFetch, request) + : createNodeTransferFetch()(request); + }, }; } @@ -562,9 +577,7 @@ function resolveProxyRoute(requestUrl: string): string { function isSupportedDaemonRoute(route: string, method: string): boolean { if (route === '/rpc') return method === 'POST'; - if (isSupportedUploadRoute(route, method)) return true; - if (route === '/artifacts' || route === '/artifacts/') return method === 'GET'; - if (route.startsWith('/artifacts/')) return method === 'GET'; + if (isTransferRoute(route, method)) return true; if (isRequestDiagnosticsRoute(route)) return method === 'GET'; return false; } @@ -586,6 +599,13 @@ function isRequestDiagnosticsRoute(route: string): boolean { ); } +function isTransferRoute(route: string, method: string): boolean { + return ( + isSupportedUploadRoute(route, method) || + (method === 'GET' && (route === '/artifacts' || route.startsWith('/artifacts/'))) + ); +} + function isSupportedUploadRoute(route: string, method: string): boolean { if (route === '/upload') return method === 'POST'; if (isUploadPreflightRoute(route)) return method === 'POST'; @@ -864,10 +884,18 @@ function rpcErrorResponse(rpcId: unknown, error: unknown): Response { ); } -function proxyErrorResponse(error: unknown): Response { +async function proxyErrorResponse(error: unknown, request: Request): Promise { + const details = isTransferRoute(resolveProxyRoute(request.url), request.method) + ? (await import('./transfer-transport.ts')).transferWireDetails(error) + : {}; const normalized = normalizeError(error); return Response.json( - { ok: false, error: normalized.message, code: normalized.code }, + { + ok: false, + error: normalized.message, + code: normalized.code, + ...details, + }, { status: normalized.code === 'INVALID_ARGS' ? 400 : 500 }, ); } diff --git a/packages/proxy/src/transfer-transport-lifetime.test.ts b/packages/proxy/src/transfer-transport-lifetime.test.ts new file mode 100644 index 0000000000..3622d8ef9d --- /dev/null +++ b/packages/proxy/src/transfer-transport-lifetime.test.ts @@ -0,0 +1,850 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import http, { type ClientRequest, type ServerResponse } from 'node:http'; +import net, { type AddressInfo } from 'node:net'; +import { expect, onTestFinished, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { createNodeTransferFetch, settleUpstreamFetch } from './transfer-transport.ts'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +function controlledClock() { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + onTestFinished(() => { + vi.useRealTimers(); + }); +} + +async function upstream(handler: http.RequestListener) { + const server = http.createServer(handler); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function observeRequest(inspect?: (request: ClientRequest) => void) { + const original = http.request; + const made = deferred(); + const closed = deferred(); + const spy = vi.spyOn(http, 'request').mockImplementation((...args) => { + const req = Reflect.apply(original, http, args) as ClientRequest; + req.once('close', () => closed.resolve()); + inspect?.(req); + made.resolve(req); + return req; + }); + onTestFinished(() => spy.mockRestore()); + return { made: made.promise, closed: closed.promise }; +} + +function phaseFailure(error: unknown, phase: string, timeoutMs: number) { + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_phase_timeout', + transferPhase: phase, + timeoutMs, + }); +} + +function emptyChunkFlood(caller: AbortController, reason: Error) { + const limit = 1_024; + let reads = 0; + let readsAtAbort = limit; + let timer: ReturnType | undefined; + const body = new ReadableStream( + { + pull(controller) { + if (reads === 0) { + timer = setTimeout(() => { + readsAtAbort = reads; + caller.abort(reason); + }, 0); + } + if (reads === limit) controller.close(); + else { + reads += 1; + controller.enqueue(new Uint8Array()); + } + }, + }, + { highWaterMark: 0 }, + ); + onTestFinished(() => clearTimeout(timer)); + return { body, limit, readsAtAbort: () => readsAtAbort }; +} + +test('empty upload chunks yield to caller cancellation before the bounded flood ends', async () => { + const origin = await upstream((req) => req.resume()); + const observed = observeRequest(); + const caller = new AbortController(); + const reason = new Error('cancel upload while empty chunks are ready'); + const flood = emptyChunkFlood(caller, reason); + const pending = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body: flood.body, + signal: caller.signal, + duplex: 'half', + } as RequestInit), + ); + await expect(pending).rejects.toBe(reason); + await observed.closed; + expect(flood.readsAtAbort()).toBeLessThan(flood.limit); + expect(flood.body.locked).toBe(false); +}); + +test('empty custom response chunks yield to caller cancellation before the bounded flood ends', async () => { + const caller = new AbortController(); + const reason = new Error('cancel response while empty chunks are ready'); + const flood = emptyChunkFlood(caller, reason); + const response = await settleUpstreamFetch( + async () => new Response(flood.body), + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + await expect(response.arrayBuffer()).rejects.toBe(reason); + expect(flood.readsAtAbort()).toBeLessThan(flood.limit); + expect(flood.body.locked).toBe(false); +}); + +test('finite empty chunks preserve the following upload and custom response bytes', async () => { + const emptyPrefix = () => + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array()); + controller.enqueue(new Uint8Array()); + controller.enqueue(new TextEncoder().encode('payload intact')); + controller.close(); + }, + }); + const origin = await upstream((req, res) => { + let text = ''; + req.setEncoding('utf8'); + req.on('data', (chunk: string) => { + text += chunk; + }); + req.on('end', () => res.end(text)); + }); + const uploaded = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body: emptyPrefix(), + duplex: 'half', + } as RequestInit), + ); + expect(await uploaded.text()).toBe('payload intact'); + const custom = await settleUpstreamFetch( + async () => new Response(emptyPrefix()), + new Request('http://daemon.internal/artifacts/build'), + ); + expect(await custom.text()).toBe('payload intact'); +}); + +test.each([401, 413, 200, 204])( + 'an early %s stops and unlocks an endless incoming reader before disposal', + async (status) => { + const origin = await upstream((_req, res) => { + res.writeHead(status, { 'content-type': 'text/plain' }); + res.end(status === 204 ? undefined : 'accepted answer'); + }); + controlledClock(); + const observed = observeRequest(); + const waiting = deferred(); + let cancelled = false; + const body = new ReadableStream( + { + start(controller) { + controller.enqueue(new Uint8Array([1])); + }, + pull() { + waiting.resolve(); + return new Promise(() => undefined); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ); + const response = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ); + await waiting.promise; + expect(response.status).toBe(status); + expect(await response.text()).toBe(status === 204 ? '' : 'accepted answer'); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(body.locked).toBe(false); + expect(cancelled).toBe(false); + expect((await observed.made).closed).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test('a progressing upload gets a fresh header ceiling at actual finish, beyond thirty seconds', async () => { + const first = deferred(); + const second = deferred(); + const receivedEnd = deferred(); + let bytes = 0; + const origin = await upstream((req) => { + req.on('data', (chunk: Buffer) => { + bytes += chunk.length; + if (bytes >= 1) first.resolve(); + if (bytes >= 2) second.resolve(); + }); + req.once('end', () => receivedEnd.resolve()); + }); + controlledClock(); + const observed = observeRequest(); + let producer!: ReadableStreamDefaultController; + const body = new ReadableStream( + { + start(controller) { + producer = controller; + }, + }, + { highWaterMark: 0 }, + ); + let settled = false; + const result = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + producer.enqueue(new Uint8Array([1])); + await first.promise; + await vi.advanceTimersByTimeAsync(31_000); + expect(settled).toBe(false); + producer.enqueue(new Uint8Array([2])); + await second.promise; + const req = await observed.made; + const finished = once(req, 'finish'); + producer.close(); + await finished; + await receivedEnd.promise; + expect(req.writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(29_999); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(req.destroyed).toBe(true); + phaseFailure(await result, 'response-headers', 30_000); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); +}); + +test('a bodyless request starts its header ceiling after actual finish on a connected socket', async () => { + const entered = deferred(); + const origin = await upstream(() => entered.resolve()); + controlledClock(); + const observed = observeRequest(); + const result = createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)).catch( + (error: unknown) => error, + ); + await entered.promise; + expect((await observed.made).writableFinished).toBe(true); + await vi.advanceTimersByTimeAsync(30_000); + expect((await observed.made).destroyed).toBe(true); + phaseFailure(await result, 'response-headers', 30_000); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); +}); + +test('response inactivity charges a pending producer, then joins its owned request close', async () => { + const origin = await upstream((_req, res) => res.flushHeaders()); + controlledClock(); + const observed = observeRequest(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + const result = response.text().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(119_999); + expect((await observed.made).destroyed).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect((await observed.made).destroyed).toBe(true); + phaseFailure(await result, 'response-body', 120_000); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(vi.getTimerCount()).toBe(0); +}); + +test('downstream backpressure suspends response inactivity without suspending the aggregate', async () => { + const peer = deferred(); + const origin = await upstream((_req, res) => { + res.flushHeaders(); + peer.resolve(res); + }); + controlledClock(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + await vi.advanceTimersByTimeAsync(240_000); + (await peer.promise).end('available on demand'); + expect(await response.text()).toBe('available on demand'); + expect(vi.getTimerCount()).toBe(0); +}); + +test('continuous response progress cannot refresh the six hundred second aggregate', async () => { + const peer = deferred(); + const origin = await upstream((_req, res) => { + res.flushHeaders(); + peer.resolve(res); + }); + controlledClock(); + const observed = observeRequest(); + const response = await createNodeTransferFetch()(new Request(`${origin}/artifacts/build`)); + const reader = response.body!.getReader(); + const res = await peer.promise; + for (let index = 0; index < 5; index += 1) { + const reading = reader.read(); + await vi.advanceTimersByTimeAsync(110_000); + res.write(Buffer.from([index + 1])); + const chunk = await reading; + expect(chunk.done).toBe(false); + expect(Array.from(chunk.value!)).toEqual([index + 1]); + } + const failure = reader.read().catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(50_000); + expect((await observed.made).destroyed).toBe(true); + const error = await failure; + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['caller', 'idle', 'aggregate'] as const)( + 'ignored foreign read and cancel cannot hold proxy settlement after %s expiry', + async (expiry) => { + controlledClock(); + const caller = new AbortController(); + const reading = deferred(); + const cancelled = deferred(); + const lateRead = deferred(); + const lateCancel = deferred(); + const foreign = new ReadableStream( + { + pull() { + reading.resolve(); + return lateRead.promise; + }, + cancel(reason) { + cancelled.resolve(reason); + return lateCancel.promise; + }, + }, + { highWaterMark: 0 }, + ); + let bounded: Request | undefined; + const response = await settleUpstreamFetch( + async (request) => { + bounded = request; + return new Response(foreign); + }, + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + if (expiry === 'aggregate') await vi.advanceTimersByTimeAsync(500_000); + let settled = false; + const result = response.text().catch((error: unknown) => error); + void result.then(() => { + settled = true; + }); + await reading.promise; + const callerReason = new AppError('COMMAND_FAILED', 'Caller stopped waiting', { + reason: 'client_gone', + }); + if (expiry === 'caller') caller.abort(callerReason); + else if (expiry === 'idle') await vi.advanceTimersByTimeAsync(120_000); + else await vi.advanceTimersByTimeAsync(100_000); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); + const error = await result; + if (expiry === 'idle') phaseFailure(error, 'response-body', 120_000); + else if (expiry === 'caller') expect(error).toBe(callerReason); + else { + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + } + expect(await cancelled.promise).toBe(error); + expect(bounded?.signal.reason).toBe(error); + expect(foreign.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + lateRead.reject(new Error('late foreign producer rejection')); + lateCancel.reject(new Error('late foreign cancel rejection')); + await new Promise((resolve) => setImmediate(resolve)); + }, +); + +test('the aggregate cancels an undemanded foreign body without joining its opaque cancel', async () => { + controlledClock(); + const cancelled = deferred(); + const response = await settleUpstreamFetch( + async () => + new Response( + new ReadableStream( + { + cancel(reason) { + cancelled.resolve(reason); + return new Promise(() => undefined); + }, + }, + { highWaterMark: 0 }, + ), + ), + new Request('http://daemon.internal/artifacts/build'), + ); + await vi.advanceTimersByTimeAsync(600_000); + const error = await response.text().catch((error: unknown) => error); + assert.ok(error instanceof AppError); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + expect(await cancelled.promise).toBe(error); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['throw', 'reject'] as const)( + 'a synchronous custom %s releases all exchange timers', + async (kind) => { + controlledClock(); + const reason = new Error('custom transport failed'); + const pending = settleUpstreamFetch(() => { + if (kind === 'throw') throw reason; + return Promise.reject(reason); + }, new Request('http://daemon.internal/artifacts/build')); + await expect(pending).rejects.toBe(reason); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test('an already aborted request never starts foreign work or leaves a timer', async () => { + controlledClock(); + const reason = new Error('already stopped'); + let calls = 0; + await expect( + settleUpstreamFetch( + async () => { + calls += 1; + return new Response('unexpected'); + }, + new Request('http://daemon.internal/artifacts/build', { signal: AbortSignal.abort(reason) }), + ), + ).rejects.toBe(reason); + expect(calls).toBe(0); + expect(vi.getTimerCount()).toBe(0); +}); + +test('late custom response and rejection outcomes stay observed after cancellation', async () => { + const unhandled: unknown[] = []; + const rejected = (error: unknown) => { + unhandled.push(error); + }; + process.on('unhandledRejection', rejected); + onTestFinished(() => { + process.off('unhandledRejection', rejected); + }); + const caller = new AbortController(); + const late = deferred(); + const cancelled = deferred(); + const pending = settleUpstreamFetch( + () => late.promise, + new Request('http://daemon.internal/artifacts/build', { signal: caller.signal }), + ); + const reason = new Error('stopped'); + caller.abort(reason); + await expect(pending).rejects.toBe(reason); + late.resolve( + new Response( + new ReadableStream({ + cancel() { + cancelled.resolve(); + }, + }), + ), + ); + await cancelled.promise; + const rejection = deferred(); + const other = new AbortController(); + const failed = settleUpstreamFetch( + () => rejection.promise, + new Request('http://daemon.internal/artifacts/build', { signal: other.signal }), + ); + other.abort(reason); + await expect(failed).rejects.toBe(reason); + rejection.reject(new Error('late tunnel failure')); + await new Promise((resolve) => setImmediate(resolve)); + expect(unhandled).toEqual([]); +}); + +test.each(['abort', 'peer-close', 'drain', 'idle', 'drain-only'] as const)( + 'an actual blocked write releases its drain wait on %s', + async (outcome) => { + const entered = deferred(); + const origin = await upstream((req, res) => { + req.pause(); + entered.resolve(req); + req.once('end', () => res.end('upload complete')); + }); + controlledClock(); + const blocked = deferred(); + let drains = 0; + const observed = observeRequest((req) => { + const onDrain = () => { + drains += 1; + }; + req.on('drain', onDrain); + onTestFinished(() => { + req.off('drain', onDrain); + }); + const write = req.write; + req.write = function (this: ClientRequest, ...args: unknown[]): boolean { + const accepted = Reflect.apply(write, this, args) as boolean; + if (!accepted) blocked.resolve(); + return accepted; + }; + }); + const caller = new AbortController(); + const body = new ReadableStream( + { + start(controller) { + controller.enqueue(new Uint8Array(32 * 1024 * 1024)); + controller.close(); + }, + }, + { highWaterMark: 0 }, + ); + const result = createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + signal: caller.signal, + duplex: 'half', + } as RequestInit), + ).catch((error: unknown) => error); + await blocked.promise; + const incoming = await entered.promise; + await new Promise((resolve) => setImmediate(resolve)); + const req = await observed.made; + expect(req.writableNeedDrain).toBe(true); + expect(req.writableLength).toBeGreaterThan(0); + expect(req.listenerCount('drain')).toBe(2); // The probe and the adapter's actual wait. + let destroyedAtOriginalDeadline = false; + const reason = new Error('client left during backpressure'); + if (outcome === 'abort') caller.abort(reason); + else if (outcome === 'peer-close') incoming.destroy(); + else if (outcome === 'idle') await vi.advanceTimersByTimeAsync(120_000); + else if (outcome === 'drain-only') { + await vi.advanceTimersByTimeAsync(60_000); + req.emit('drain'); + expect(req.writableLength).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); + destroyedAtOriginalDeadline = req.destroyed; + await vi.advanceTimersByTimeAsync(120_000); + } else incoming.resume(); + const response = await result; + if (outcome === 'drain') { + assert.ok(response instanceof Response); + expect(await response.text()).toBe('upload complete'); + expect(drains).toBeGreaterThan(0); + } else if (outcome === 'abort') expect(response).toBe(reason); + else if (outcome === 'idle' || outcome === 'drain-only') + phaseFailure(response, 'request-body', 120_000); + else expect(response).toBeInstanceOf(Error); + await observed.closed; + await new Promise((resolve) => setImmediate(resolve)); + expect(req.listenerCount('drain')).toBe(1); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + if (outcome === 'drain-only') expect(destroyedAtOriginalDeadline).toBe(true); + }, +); + +test('a completed exchange leaves its global-agent socket available to the next request', async () => { + const sockets: http.IncomingMessage['socket'][] = []; + const origin = await upstream((req, res) => { + sockets.push(req.socket); + res.end('pooled response'); + }); + const requests: ClientRequest[] = []; + observeRequest((req) => { + requests.push(req); + }); + const send = createNodeTransferFetch(); + expect(await (await send(new Request(`${origin}/artifacts/first`))).text()).toBe( + 'pooled response', + ); + expect(sockets[0]?.destroyed).toBe(false); + expect(await (await send(new Request(`${origin}/artifacts/second`))).text()).toBe( + 'pooled response', + ); + expect(requests[1]?.reusedSocket).toBe(true); + expect(sockets[1]).toBe(sockets[0]); + expect(sockets[1]?.destroyed).toBe(false); +}); + +test('a real TCP connection without a completed TLS handshake expires in the connect phase', async () => { + const accepted = deferred(); + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on('error', () => undefined); + socket.once('close', () => sockets.delete(socket)); + accepted.resolve(); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + onTestFinished(async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }); + controlledClock(); + const port = (server.address() as AddressInfo).port; + const result = createNodeTransferFetch()( + new Request(`https://127.0.0.1:${port}/artifacts/build`), + ).catch((error: unknown) => error); + await accepted.promise; + await vi.advanceTimersByTimeAsync(30_000); + phaseFailure(await result, 'connect', 30_000); + expect(vi.getTimerCount()).toBe(0); +}); + +test('downstream body cancellation aborts the custom request without joining an ignored cancel callback', async () => { + controlledClock(); + const reading = deferred(); + const cancelled = deferred(); + let bounded!: Request; + const body = new ReadableStream( + { + pull() { + reading.resolve(); + return new Promise(() => undefined); + }, + cancel(reason) { + cancelled.resolve(reason); + return new Promise(() => undefined); + }, + }, + { highWaterMark: 0 }, + ); + const response = await settleUpstreamFetch(async (request) => { + bounded = request; + return new Response(body); + }, new Request('http://daemon.internal/artifacts/build')); + const reader = response.body!.getReader(); + const pending = reader.read(); + await reading.promise; + const reason = new Error('downstream stopped reading'); + await reader.cancel(reason); + expect(await pending).toEqual({ done: true, value: undefined }); + expect(await cancelled.promise).toBe(reason); + expect(bounded.signal.reason).toBe(reason); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(['iterator stop', 'request close', 'outgoing pump'] as const)( + 'owned cleanup awaits %s before settlement and timer disposal', + async (held) => { + const origin = await upstream((_req, res) => { + res.setHeader('connection', 'close'); + res.end('joined'); + }); + controlledClock(); + const stopEntered = deferred(); + const responseClosed = deferred(); + const pumpUnlocked = deferred(); + const pumpStopEntered = deferred(); + const allowPump = deferred(); + const allowStop = deferred(); + const closeEntered = deferred(); + const allowClose = deferred(); + onTestFinished(() => { + allowStop.resolve(); + allowClose.resolve(); + allowPump.resolve(); + }); + let unlocked = false; + const observed = observeRequest((req) => { + const once = req.once; + const closeRegistration = vi.spyOn(req, 'once').mockImplementation((event, listener) => { + if (event !== 'close') return Reflect.apply(once, req, [event, listener]); + closeRegistration.mockRestore(); + return Reflect.apply(once, req, [ + event, + (...args: unknown[]) => { + closeEntered.resolve(); + if (held === 'request close') + void allowClose.promise.then(() => Reflect.apply(listener, req, args)); + else Reflect.apply(listener, req, args); + }, + ]); + }); + onTestFinished(() => closeRegistration.mockRestore()); + req.prependListener('response', (res) => { + res.once('close', () => responseClosed.resolve()); + const iterate = res[Symbol.asyncIterator].bind(res); + res[Symbol.asyncIterator] = () => { + const iterator = iterate(); + const stop = iterator.return!.bind(iterator); + iterator.return = async (value) => { + const stopped = await stop(value); + stopEntered.resolve(); + if (held === 'iterator stop') await allowStop.promise; + return stopped; + }; + return iterator; + }; + }); + if (held === 'outgoing pump') queueMicrotask(() => req.flushHeaders()); + }); + const body = new ReadableStream( + { + start(controller) { + if (held !== 'outgoing pump') controller.enqueue(new Uint8Array([1])); + }, + }, + { highWaterMark: 0 }, + ); + const getReader = body.getReader.bind(body); + body.getReader = (() => { + const reader = getReader(); + const release = reader.releaseLock.bind(reader); + reader.releaseLock = () => { + release(); + unlocked = true; + pumpUnlocked.resolve(); + }; + if (held === 'outgoing pump') { + const add = AbortSignal.prototype.addEventListener; + const registration = vi + .spyOn(AbortSignal.prototype, 'addEventListener') + .mockImplementation(function (this: AbortSignal, event, listener, options) { + registration.mockRestore(); + expect(event).toBe('abort'); + return Reflect.apply(add, this, [ + event, + (value: Event) => { + pumpStopEntered.resolve(); + void allowPump.promise.then(() => { + if (typeof listener === 'function') listener.call(this, value); + else listener?.handleEvent(value); + }); + }, + options, + ]); + }); + onTestFinished(() => registration.mockRestore()); + } + return reader; + }) as typeof body.getReader; + const response = await createNodeTransferFetch()( + new Request(`${origin}/upload/direct/ticket`, { + method: 'PUT', + body, + duplex: 'half', + } as RequestInit), + ); + let settled = false; + const result = response.text().then((text) => { + settled = true; + return text; + }); + await { + 'iterator stop': stopEntered.promise, + 'request close': closeEntered.promise, + 'outgoing pump': pumpStopEntered.promise, + }[held]; + await Promise.all([observed.closed, responseClosed.promise]); + if (held !== 'outgoing pump') await pumpUnlocked.promise; + await new Promise((resolve) => setImmediate(resolve)); + expect(unlocked).toBe(held !== 'outgoing pump'); + expect(settled).toBe(false); + expect(vi.getTimerCount()).toBeGreaterThan(0); + allowStop.resolve(); + allowClose.resolve(); + allowPump.resolve(); + expect(await result).toBe('joined'); + await pumpUnlocked.promise; + expect(unlocked).toBe(true); + await observed.closed; + expect(vi.getTimerCount()).toBe(0); + }, +); + +test.each(['phase expiry', 'caller abort'] as const)( + 'a failure survives %s during joined cleanup', + async (late) => { + const entered = deferred(); + const origin = await upstream(() => entered.resolve()); + controlledClock(); + const closeEntered = deferred(); + const allowClose = deferred(); + onTestFinished(() => allowClose.resolve()); + const observed = observeRequest((req) => { + const once = req.once; + const registration = vi.spyOn(req, 'once').mockImplementation((event, listener) => { + if (event !== 'close') return Reflect.apply(once, req, [event, listener]); + registration.mockRestore(); + return Reflect.apply(once, req, [ + event, + (...args: unknown[]) => { + closeEntered.resolve(); + void allowClose.promise.then(() => Reflect.apply(listener, req, args)); + }, + ]); + }); + onTestFinished(() => registration.mockRestore()); + }); + const caller = new AbortController(); + const original = new AppError('COMMAND_FAILED', 'upstream failed before cleanup', { + reason: 'transfer_upstream_closed', + }); + let settled = false; + const result = createNodeTransferFetch()( + new Request(`${origin}/artifacts/build`, { signal: caller.signal }), + ).catch((error: unknown) => { + settled = true; + return error; + }); + await entered.promise; + (await observed.made).destroy(original); + await closeEntered.promise; + expect(settled).toBe(false); + if (late === 'phase expiry') await vi.advanceTimersByTimeAsync(30_000); + else caller.abort(new Error('later caller abort')); + allowClose.resolve(); + expect(await result).toBe(original); + expect(vi.getTimerCount()).toBe(0); + }, +); diff --git a/packages/proxy/src/transfer-transport.test.ts b/packages/proxy/src/transfer-transport.test.ts new file mode 100644 index 0000000000..feff19b307 --- /dev/null +++ b/packages/proxy/src/transfer-transport.test.ts @@ -0,0 +1,374 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import type { IncomingMessage } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterAll, expect, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { isTransferPhaseTimeoutError } from '@agent-device/host-kit/transport'; +import { + createNodeTransferFetch, + settleUpstreamFetch, + transferWireDetails, +} from './transfer-transport.ts'; + +const servers: http.Server[] = []; + +afterAll(async () => { + await Promise.all( + servers.map( + (server) => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); +}); + +function startUpstream(handler: http.RequestListener): Promise { + const server = http.createServer(handler); + servers.push(server); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${port}`); + }); + }); +} + +function transferFetch() { + return createNodeTransferFetch(); +} + +function streamedBody(totalBytes: number, chunkBytes: number): ReadableStream { + let sent = 0; + return new ReadableStream({ + pull(controller) { + if (sent >= totalBytes) { + controller.close(); + return; + } + const size = Math.min(chunkBytes, totalBytes - sent); + sent += size; + controller.enqueue(new Uint8Array(size).fill(0x61)); + }, + }); +} + +test('a location-less 308 chunk answer keeps its resume offsets', async () => { + const origin = await startUpstream((_req, res) => { + res.writeHead(308, { range: 'bytes=0-1023', 'x-upload-offset': '1024' }); + res.end(); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(1024)); + controller.close(); + }, + }), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(308); + expect(response.headers.get('x-upload-offset')).toBe('1024'); + expect(response.headers.get('range')).toBe('bytes=0-1023'); + expect(response.headers.get('location')).toBeNull(); + await response.text(); +}); + +test('a redirect with a Location is answered as a status, never followed', async () => { + const followed = await startUpstream((_req, res) => { + res.statusCode = 200; + res.end('followed'); + }); + const origin = await startUpstream((_req, res) => { + res.writeHead(307, { location: `${followed}/artifacts/art-1` }); + res.end(); + }); + + const response = await transferFetch()( + new Request(`${origin}/artifacts/art-1`, { method: 'GET' }), + ); + + expect(response.status).toBe(307); + expect(await response.text()).toBe(''); +}); + +test('an early 413 keeps its own status against a later peer break', async () => { + let upstreamRequest!: IncomingMessage; + const origin = await startUpstream((req, res) => { + upstreamRequest = req; + req.on('error', () => undefined); + res.statusCode = 413; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ ok: false, error: 'Artifact upload is too large' })); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: new ReadableStream({ + start(controller) { + for (let i = 0; i < 32; i += 1) controller.enqueue(new Uint8Array(64 * 1024)); + }, + }), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(413); + upstreamRequest.destroy(); + expect((await response.json()) as unknown).toEqual({ + ok: false, + error: 'Artifact upload is too large', + }); +}); + +test('an upload that is read completely answers with its status and streamed body', async () => { + const origin = await startUpstream(async (req, res) => { + let received = 0; + for await (const chunk of req) received += (chunk as Buffer).byteLength; + res.statusCode = 200; + res.end(`received:${received}`); + }); + + const response = await transferFetch()( + new Request(`${origin}/upload/direct/abc`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: streamedBody(256 * 1024, 16 * 1024), + duplex: 'half', + } as RequestInit), + ); + + expect(response.status).toBe(200); + expect(await response.text()).toBe('received:262144'); +}); + +test('a download body is handed over in order and ends exactly once', async () => { + const origin = await startUpstream((_req, res) => { + res.statusCode = 200; + res.setHeader('content-type', 'application/octet-stream'); + res.write(Buffer.alloc(1024, 0x62)); + res.write(Buffer.alloc(1024, 0x63)); + res.end(); + }); + + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + + expect(response.status).toBe(200); + const body = new Uint8Array(await response.arrayBuffer()); + expect(body).toHaveLength(2048); + expect(body[0]).toBe(0x62); + expect(body[1024]).toBe(0x63); +}); + +test('a null-body status and a HEAD answer carry headers with no body', async () => { + const origin = await startUpstream((req, res) => { + if (req.method === 'HEAD') { + res.writeHead(200, { 'content-length': '4096' }); + res.end(); + return; + } + res.writeHead(204, { 'x-artifact': 'gone' }); + res.end(); + }); + const fetchTransfer = transferFetch(); + + const head = await fetchTransfer(new Request(`${origin}/artifacts/art-1`, { method: 'HEAD' })); + const noContent = await fetchTransfer(new Request(`${origin}/artifacts/art-1`)); + + expect(head.status).toBe(200); + expect(head.headers.get('content-length')).toBe('4096'); + expect(head.body).toBeNull(); + expect(noContent.status).toBe(204); + expect(noContent.headers.get('x-artifact')).toBe('gone'); + expect(noContent.body).toBeNull(); +}); + +test('caller cancellation is reported with the caller reason verbatim', async () => { + const origin = await startUpstream(() => undefined); + const caller = new AbortController(); + const cancellation = new Error('caller went away'); + + const pending = transferFetch()( + new Request(`${origin}/artifacts/art-1`, { signal: caller.signal }), + ); + caller.abort(cancellation); + + await expect(pending).rejects.toBe(cancellation); +}); + +test('an upstream that dies before answering fails without a phase reason', async () => { + const origin = await startUpstream((req) => { + req.destroy(); + }); + + await expect(transferFetch()(new Request(`${origin}/artifacts/art-1`))).rejects.toBeInstanceOf( + Error, + ); +}); + +test('a forwarded request carries its method, path, query and allowed headers', async () => { + let seen: IncomingMessage | undefined; + const origin = await startUpstream((req, res) => { + seen = req; + res.statusCode = 200; + res.end('ok'); + }); + + const response = await transferFetch()( + new Request(`${origin}/artifacts/art-1?version=2`, { + headers: { 'x-artifact-hash': 'sha256:abc', authorization: 'Bearer daemon-secret' }, + }), + ); + await response.text(); + + expect(seen?.method).toBe('GET'); + expect(seen?.url).toBe('/artifacts/art-1?version=2'); + expect(seen?.headers['x-artifact-hash']).toBe('sha256:abc'); + expect(seen?.headers.authorization).toBe('Bearer daemon-secret'); +}); + +test('the default transport injects neither Fetch headers nor URL authorization', async () => { + let headers: http.IncomingHttpHeaders = {}; + const origin = await startUpstream((req, res) => { + headers = req.headers; + res.end('ok'); + }); + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + expect(await response.text()).toBe('ok'); + expect(headers.authorization).toBeUndefined(); + expect(headers.accept).toBeUndefined(); + expect(headers['user-agent']).toBeUndefined(); +}); + +test('a custom transport keeps its response body for the caller', async () => { + const response = await settleUpstreamFetch( + () => + Promise.resolve( + new Response('{"ok":true}', { + status: 201, + headers: { 'content-type': 'application/json' }, + }), + ), + new Request('http://daemon.internal:4310/upload/preflight', { method: 'POST' }), + ); + + expect(response.status).toBe(201); + expect(response.headers.get('content-type')).toBe('application/json'); + expect(await response.text()).toBe('{"ok":true}'); +}); + +test('the default transport serves an unchunked body whose length is known up front', async () => { + const payload = Buffer.alloc(4096, 0x7a); + const origin = await startUpstream((_req, res) => { + res.writeHead(200, { 'content-length': String(payload.byteLength) }); + res.end(payload); + }); + + const response = await transferFetch()(new Request(`${origin}/artifacts/art-1`)); + + expect(response.status).toBe(200); + const body = new Uint8Array(await response.arrayBuffer()); + expect(body).toHaveLength(4096); + expect(body.every((byte) => byte === 0x7a)).toBe(true); +}); + +test('the forwarding aggregate ends a transport that never settles and names its phase', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const pending = settleUpstreamFetch( + () => new Promise(() => undefined), + new Request('http://daemon.internal:4310/artifacts/art-1'), + ); + let settled = false; + const failure = pending.catch((error: unknown) => error); + void failure.then(() => { + settled = true; + }); + await vi.advanceTimersByTimeAsync(600_000 + 1); + expect(settled).toBe(true); + const error = await failure; + assert.ok(error instanceof AppError); + assert.equal(isTransferPhaseTimeoutError(error), false); + expect(error.details).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + } finally { + vi.useRealTimers(); + } +}); + +test('a transfer failure exposes the adopted wire details and nothing else does', () => { + const phaseFailure = new AppError( + 'COMMAND_FAILED', + 'Artifact transfer response-headers timed out', + { + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }, + ); + + expect(transferWireDetails(phaseFailure)).toEqual({ + reason: 'transfer_phase_timeout', + transferPhase: 'response-headers', + timeoutMs: 30_000, + }); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }), + ), + ).toEqual({ + reason: 'transfer_forwarding_deadline_exceeded', + transferPhase: 'forwarding', + timeoutMs: 600_000, + }); + expect( + transferWireDetails(new AppError('UNAUTHORIZED', 'Invalid proxy token', { reason: 'other' })), + ).toEqual({}); + expect(transferWireDetails(new Error('socket hang up'))).toEqual({}); +}); + +test('a transfer failure with unusable details stays off the wire', () => { + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: '', + timeoutMs: 30_000, + }), + ), + ).toEqual({}); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + timeoutMs: Number.NaN, + }), + ), + ).toEqual({}); + expect( + transferWireDetails( + new AppError('COMMAND_FAILED', 'masked', { + reason: 'transfer_phase_timeout', + transferPhase: 'response-body', + }), + ), + ).toEqual({}); +}); diff --git a/packages/proxy/src/transfer-transport.ts b/packages/proxy/src/transfer-transport.ts new file mode 100644 index 0000000000..e2849fa2c9 --- /dev/null +++ b/packages/proxy/src/transfer-transport.ts @@ -0,0 +1,472 @@ +import type { ClientRequest, IncomingMessage, RequestOptions } from 'node:http'; +import { AppError } from '@agent-device/kernel/errors'; +import { + createTransferPhaseWatchdog, + isTransferPhaseTimeoutError, + loadNodeHttpRequester, + observeClientRequestPhases, + observeCompletedRequestWrites, + TRANSFER_BODY_INACTIVITY_TIMEOUT_MS, + TRANSFER_CONNECT_TIMEOUT_MS, + TRANSFER_RESPONSE_HEADER_TIMEOUT_MS, + type TransferPhaseWatchdog, +} from '@agent-device/host-kit/transport'; + +const PROXY_TRANSFER_AGGREGATE_MS = 600_000; +const TRANSFER_FORWARDING_DEADLINE_REASON = 'transfer_forwarding_deadline_exceeded'; +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + +/** Forwards one exchange over the runtime's default global Node agent, without redirects or retries. */ +export function createNodeTransferFetch(): (request: Request) => Promise { + return async (request) => { + const exchange = openExchange(request.signal); + try { + return await exchangeOverNodeHttp(request, exchange); + } catch (error) { + exchange.dispose(); + throw error; + } + }; +} + +/** Bounds proxy-owned waits; opaque transport work and cancellation callbacks are never joined. */ +export async function settleUpstreamFetch( + upstreamFetch: (request: Request) => Promise, + request: Request, +): Promise { + const exchange = openExchange(request.signal); + let response: Response | undefined; + try { + response = await untilAborted( + exchange.signal, + () => upstreamFetch(new Request(request, { signal: exchange.signal })), + (late) => cancelForeignBody(late.body), + ); + if (!response.body) { + exchange.dispose(); + return response; + } + return new Response( + demandAwareStream(foreignBodySource(response.body), exchange, () => { + exchange.dispose(); + }), + { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }, + ); + } catch (error) { + const failure = exchange.classify(error); + cancelForeignBody(response?.body ?? null); + exchange.dispose(); + throw failure; + } +} + +export function transferWireDetails(error: unknown): Record { + if (!(error instanceof AppError) || error.code !== 'COMMAND_FAILED') return {}; + const { reason, transferPhase, timeoutMs } = error.details ?? {}; + if (reason === TRANSFER_FORWARDING_DEADLINE_REASON) { + return transferPhase === 'forwarding' && timeoutMs === PROXY_TRANSFER_AGGREGATE_MS + ? { reason, transferPhase, timeoutMs } + : {}; + } + if (!isTransferPhaseTimeoutError(error)) return {}; + let phaseBudgetMs: number; + switch (transferPhase) { + case 'connect': + phaseBudgetMs = TRANSFER_CONNECT_TIMEOUT_MS; + break; + case 'response-headers': + phaseBudgetMs = TRANSFER_RESPONSE_HEADER_TIMEOUT_MS; + break; + case 'request-body': + case 'response-body': + phaseBudgetMs = TRANSFER_BODY_INACTIVITY_TIMEOUT_MS; + break; + default: + return {}; + } + return timeoutMs === phaseBudgetMs ? { reason, transferPhase, timeoutMs } : {}; +} + +type TransferExchange = { + signal: AbortSignal; + watchdog: TransferPhaseWatchdog; + classify(error: unknown): unknown; + cancel(reason?: unknown): void; + dispose(): void; +}; + +function openExchange(caller: AbortSignal): TransferExchange { + const forwarding = new AbortController(); + const timer = setTimeout( + () => + forwarding.abort( + new AppError('COMMAND_FAILED', 'Proxy transfer forwarding deadline exceeded', { + reason: TRANSFER_FORWARDING_DEADLINE_REASON, + transferPhase: 'forwarding', + timeoutMs: PROXY_TRANSFER_AGGREGATE_MS, + }), + ), + PROXY_TRANSFER_AGGREGATE_MS, + ); + timer.unref(); + const watchdog = createTransferPhaseWatchdog(AbortSignal.any([caller, forwarding.signal])); + return { + signal: watchdog.signal, + watchdog, + classify: (error) => (watchdog.signal.aborted ? watchdog.signal.reason : error), + cancel: (reason) => forwarding.abort(reason), + dispose: () => { + clearTimeout(timer); + watchdog.dispose(); + }, + }; +} + +async function exchangeOverNodeHttp( + request: Request, + exchange: TransferExchange, +): Promise { + const url = new URL(request.url); + const [transport, { urlToHttpOptions }] = await untilAborted(exchange.signal, () => + Promise.all([loadNodeHttpRequester(url.protocol), import('node:url')]), + ); + exchange.signal.throwIfAborted(); + const { auth: _auth, ...target } = urlToHttpOptions(url); + const options: RequestOptions = { + ...target, + method: request.method, + path: `${url.pathname}${url.search}`, + headers: Object.fromEntries(request.headers), + signal: exchange.signal, + }; + exchange.watchdog.arm('connect', TRANSFER_CONNECT_TIMEOUT_MS); + return await new Promise((resolve, reject) => { + const outgoingStopped = new AbortController(); + const requestClosed = new AbortController(); + let answered = false; + let response: IncomingMessage | undefined; + let responseClosed: Promise = Promise.resolve(); + let finishing: Promise | undefined; + const finish = () => { + if (finishing) return finishing; + outgoingStopped.abort(); + // A flushed request can already have returned its socket to the global pool. + if (!req.writableFinished && !req.closed) req.destroy(); + finishing = Promise.all([pump, reqClosed, responseClosed]).then(() => { + phases.dispose(); + stopObservingWrites(); + req.off('error', fail); + response?.off('error', fail); + exchange.signal.removeEventListener('abort', onAbort); + exchange.dispose(); + }); + return finishing; + }; + const fail = (error: unknown) => { + if (answered) return; + answered = true; + const failure = exchange.classify(error); + response?.destroy(); + req.destroy(); + void finish().then(() => reject(failure)); + }; + const req = transport.request(options, (res) => { + response = res; + responseClosed = new Promise((closed) => res.once('close', closed)); + res.on('error', fail); + if (answered || exchange.signal.aborted) { + res.destroy(); + fail(exchange.signal.reason); + return; + } + const status = res.statusCode ?? 500; + if (status < 200 || status > 599) { + fail( + new AppError('COMMAND_FAILED', 'Upstream daemon returned an unusable HTTP status', { + reason: 'transfer_gateway_status', + status, + }), + ); + return; + } + answered = true; + outgoingStopped.abort(); + exchange.watchdog.suspend(); + const headers = new Headers(); + for (const [name, value] of Object.entries(res.headers)) { + if (value === undefined) continue; + for (const item of Array.isArray(value) ? value : [value]) headers.append(name, item); + } + if (request.method === 'HEAD' || NULL_BODY_STATUSES.has(status)) { + res.resume(); + void responseClosed.then(finish); + resolve(new Response(null, { status, headers })); + return; + } + resolve( + new Response(demandAwareStream(nodeBodySource(res), exchange, finish), { + status, + headers, + }), + ); + }); + const reqClosed = new Promise((closed) => { + req.once('close', () => { + requestClosed.abort(); + closed(); + if (!answered) + fail( + new AppError('COMMAND_FAILED', 'Upstream daemon closed before responding', { + reason: 'transfer_upstream_closed', + }), + ); + }); + }); + req.on('error', fail); + const phases = observeClientRequestPhases(req, exchange.watchdog); + const stopObservingWrites = observeCompletedRequestWrites(req, exchange.signal, () => { + exchange.watchdog.touchBody(); + }); + const onAbort = () => { + response?.destroy(); + fail(exchange.signal.reason); + }; + exchange.signal.addEventListener('abort', onAbort, { once: true }); + const pump = pumpOutgoing( + request, + req, + exchange.signal, + AbortSignal.any([exchange.signal, outgoingStopped.signal, requestClosed.signal]), + ).catch(fail); + if (exchange.signal.aborted) onAbort(); + }); +} + +async function pumpOutgoing( + request: Request, + req: ClientRequest, + lifetime: AbortSignal, + stopped: AbortSignal, +): Promise { + if (stopped.aborted) return; + if (!request.body) { + req.end(); + return; + } + const reader = request.body.getReader(); + try { + for (;;) { + const { done, value } = await untilAborted(stopped, () => reader.read()); + if (stopped.aborted) return; + if (done) { + req.end(); + return; + } + if (value.byteLength === 0) { + await yieldTransferEvents(stopped); + continue; + } + if (!req.write(value)) await waitUntilDrained(req, stopped); + } + } catch (error) { + if (!stopped.aborted) throw error; + } finally { + // Releasing a pending read rejects that read without cancelling the incoming client socket. + if (lifetime.aborted) void reader.cancel(lifetime.reason).catch(() => undefined); + reader.releaseLock(); + } +} + +function waitUntilDrained(req: ClientRequest, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const cleanup = () => { + req.off('drain', drained); + signal.removeEventListener('abort', aborted); + }; + const drained = () => { + cleanup(); + resolve(); + }; + const aborted = () => { + cleanup(); + reject(signal.reason); + }; + req.once('drain', drained); + signal.addEventListener('abort', aborted, { once: true }); + if (signal.aborted) aborted(); + else if (req.destroyed || req.closed) { + cleanup(); + reject( + new AppError('COMMAND_FAILED', 'Upstream daemon closed during a write', { + reason: 'transfer_upstream_closed', + }), + ); + } else if (!req.writableNeedDrain) drained(); + }); +} + +type BodySource = { + next(): Promise; + stop(reason?: unknown): void | Promise; +}; + +function foreignBodySource(body: ReadableStream): BodySource { + const reader = body.getReader(); + return { + next: async () => { + const { done, value } = await reader.read(); + return done ? null : value; + }, + stop: (reason) => { + void reader.cancel(reason).catch(() => undefined); + reader.releaseLock(); + }, + }; +} + +function cancelForeignBody(body: ReadableStream | null): void { + if (body) void body.cancel().catch(() => undefined); +} + +function nodeBodySource(res: IncomingMessage): BodySource { + const iterator = res[Symbol.asyncIterator](); + return { + next: async () => { + const { done, value } = await iterator.next(); + return done ? null : value; + }, + stop: async () => { + if (!res.destroyed) res.destroy(); + await iterator.return?.(); + }, + }; +} + +function demandAwareStream( + source: BodySource, + exchange: TransferExchange, + cleanup: () => void | Promise, +): ReadableStream { + let ended = false; + let finishing: Promise | undefined; + const finish = (reason?: unknown) => { + if (finishing) return finishing; + ended = true; + exchange.signal.removeEventListener('abort', onAbort); + exchange.watchdog.suspend(); + finishing = Promise.resolve(source.stop(reason)) + .catch(() => undefined) + .then(cleanup); + return finishing; + }; + let downstream: ReadableStreamDefaultController; + const onAbort = () => { + if (ended) return; + downstream.error(exchange.signal.reason); + void finish(exchange.signal.reason); + }; + return new ReadableStream( + { + start(controller) { + downstream = controller; + exchange.signal.addEventListener('abort', onAbort, { once: true }); + if (exchange.signal.aborted) onAbort(); + }, + async pull(controller) { + if (ended) return; + exchange.watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS); + try { + for (;;) { + const chunk = await untilAborted(exchange.signal, () => source.next()); + if (ended) return; + if (chunk === null) { + await finish(); + controller.close(); + return; + } + if (chunk.byteLength === 0) { + await yieldTransferEvents(exchange.signal); + continue; + } + exchange.watchdog.suspend(); + controller.enqueue(chunk); + return; + } + } catch (error) { + if (ended) return; + const failure = exchange.classify(error); + controller.error(failure); + await finish(failure); + } + }, + cancel(reason) { + exchange.cancel(reason); + return finish(reason); + }, + }, + { highWaterMark: 0 }, + ); +} + +function yieldTransferEvents(signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener('abort', aborted); + resolve(); + }, 0); + const aborted = () => { + clearTimeout(timer); + signal.removeEventListener('abort', aborted); + reject(signal.reason); + }; + signal.addEventListener('abort', aborted, { once: true }); + if (signal.aborted) aborted(); + }); +} + +function untilAborted( + signal: AbortSignal, + run: () => Promise, + discardLate?: (value: T) => void, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const stop = () => { + settled = true; + signal.removeEventListener('abort', aborted); + }; + const aborted = () => { + stop(); + reject(signal.reason); + }; + if (signal.aborted) { + aborted(); + return; + } + signal.addEventListener('abort', aborted, { once: true }); + try { + run().then( + (value) => { + if (settled) { + discardLate?.(value); + return; + } + stop(); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + stop(); + reject(error); + }, + ); + } catch (error) { + stop(); + reject(error); + } + }); +}