Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions packages/daemon/src/server/http/http-server-proxy-rollback.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
import assert from 'node:assert/strict';
import crypto from 'node:crypto';
import fs from 'node:fs';
import http from 'node:http';
import { expect, onTestFinished, test, vi } from 'vitest';
import { createDaemonProxyServer } from '@agent-device/proxy';
import {
closeLoopbackServer,
createTestRuntimeStateHandle,
listenOnLoopback,
skipWhenLoopbackUnavailable,
} from '@agent-device/test-kit';
import {
cleanupUploadedArtifact,
prepareUploadedArtifact,
} from '../../resources/artifact-tracking.ts';
import { createDaemonHttpServer } from './http-server.ts';

function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (error: unknown) => void;
const promise = new Promise<T>((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}

test.for(['disconnect and resume', 'uninterrupted'] as const)(
'an actual proxy-to-daemon upload handles %s and verifies finalized bytes',
async (mode, t) => {
if (await skipWhenLoopbackUnavailable(t, 'proxy-to-daemon upload coverage')) return;
const runtime = createTestRuntimeStateHandle();
try {
await runtime.run(async () => {
const daemon = await createDaemonHttpServer({
token: 'daemon-secret',
handleRequest: async () => ({ ok: true, data: {} }),
env: process.env,
cwd: process.cwd(),
});
const daemonPort = await listenOnLoopback(daemon);
const proxy = createDaemonProxyServer({
upstreamBaseUrl: `http://127.0.0.1:${daemonPort}`,
upstreamToken: 'daemon-secret',
clientToken: 'client-secret',
});
let tracked = '';
try {
const origin = `http://127.0.0.1:${await listenOnLoopback(proxy)}`;
const content = Buffer.from('ABCDEF');
const headers = {
authorization: 'Bearer client-secret',
'content-type': 'application/json',
};
const preflight = await fetch(`${origin}/upload/preflight`, {
method: 'POST',
headers,
body: JSON.stringify({
uploadAttemptId: crypto.randomUUID(),
sha256: crypto.createHash('sha256').update(content).digest('hex'),
fileName: 'demo.apk',
sizeBytes: content.length,
artifactType: 'file',
platform: 'android',
}),
});
expect(preflight.status).toBe(200);
const ticket = (await preflight.json()) as {
uploadId: string;
upload: { url: string; headers: Record<string, string> };
};
expect(new URL(ticket.upload.url).origin).toBe(origin);
expect(ticket.upload.headers.authorization).toBe('Bearer client-secret');
const put = (range: string, body: string) =>
fetch(ticket.upload.url, {
method: 'PUT',
headers: { ...ticket.upload.headers, 'content-range': range },
body,
});
const first = await put('bytes 0-1/6', 'AB');
expect(first.status).toBe(308);
expect(first.headers.get('x-upload-offset')).toBe('2');
await first.text();
if (mode === 'disconnect and resume') {
const prefix = deferred<string>();
const rolledBack = deferred<void>();
let payload = '';
let watcher: fs.FSWatcher | undefined;
const createWriter = fs.createWriteStream;
const writerSpy = vi.spyOn(fs, 'createWriteStream').mockImplementation((...args) => {
const writer = Reflect.apply(createWriter, fs, args) as fs.WriteStream;
payload = String(args[0]);
writer.once('open', () => {
watcher = fs.watch(payload, () => {
void fs.promises.readFile(payload, 'utf8').then((bytes) => {
if (bytes === 'ABCD') prefix.resolve(bytes);
}, prefix.reject);
});
});
return writer;
});
const truncate = fs.promises.truncate;
const rollbackSpy = vi
.spyOn(fs.promises, 'truncate')
.mockImplementation(async (path, length) => {
await truncate(path, length);
if (String(path) === payload && length === 2) rolledBack.resolve();
});
onTestFinished(() => {
watcher?.close();
writerSpy.mockRestore();
rollbackSpy.mockRestore();
});
const client = http.request(ticket.upload.url, {
method: 'PUT',
headers: { ...ticket.upload.headers, 'content-range': 'bytes 2-5/6' },
});
const errors: Error[] = [];
client.on('error', (error) => errors.push(error));
onTestFinished(() => {
client.destroy();
});
client.write('CD');
expect(await prefix.promise).toBe('ABCD');
assert.equal(fs.readFileSync(payload, 'utf8'), 'ABCD');
const closed = new Promise<void>((resolve) => client.once('close', () => resolve()));
client.destroy();
await closed;
await rolledBack.promise;
expect(fs.readFileSync(payload, 'utf8')).toBe('AB');
expect(client.writableEnded).toBe(false);
expect(errors.length).toBeGreaterThan(0);
expect(
errors.every((error) => (error as NodeJS.ErrnoException).code === 'ECONNRESET'),
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
).toBe(true);
watcher?.close();
writerSpy.mockRestore();
rollbackSpy.mockRestore();
const probe = await put('bytes 0-1/6', 'AB');
expect(probe.status).toBe(308);
expect(probe.headers.get('x-upload-offset')).toBe('2');
await probe.text();
}
const complete = await put('bytes 2-5/6', 'CDEF');
expect(complete.status).toBe(200);
await complete.text();
const finalize = await fetch(`${origin}/upload/finalize`, {
method: 'POST',
headers,
body: JSON.stringify({ uploadId: ticket.uploadId }),
});
expect(finalize.status).toBe(200);
tracked = ((await finalize.json()) as { uploadId: string }).uploadId;
expect(fs.readFileSync(prepareUploadedArtifact(tracked))).toEqual(content);
} finally {
if (tracked) cleanupUploadedArtifact(tracked);
await closeLoopbackServer(proxy);
await closeLoopbackServer(daemon);
}
});
} finally {
runtime.dispose();
}
},
);
32 changes: 32 additions & 0 deletions packages/host-kit/src/internal/transport-phases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,38 @@ test('disposing the phase observer removes only its own listeners', () => {
assert.equal(fake.listenerCount('response'), 0);
});

test('suspend resumes counting when the same phase is armed again', async () => {
const watchdog = watchPhases();
watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS);
await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS - 10_000);
watchdog.suspend();
await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 3);
assert.equal(watchdog.signal.aborted, false);
watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS);
await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS + 1);
assertPhaseDetails(watchdog.signal.reason, 'response-body', 120_000);
});

test('body progress during a suspend cannot restart the stopped ceiling', async () => {
const watchdog = watchPhases();
watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS);
watchdog.suspend();
watchdog.touchBody();
await vi.advanceTimersByTimeAsync(TRANSFER_BODY_INACTIVITY_TIMEOUT_MS * 2);
assert.equal(watchdog.signal.aborted, false);
});

test('caller cancellation still binds an exchange whose idle ceiling is suspended', async () => {
const caller = new AbortController();
const watchdog = watchPhases(caller.signal);
watchdog.arm('response-body', TRANSFER_BODY_INACTIVITY_TIMEOUT_MS);
watchdog.suspend();
const cancellation = new Error('caller went away');
caller.abort(cancellation);
assert.equal(watchdog.signal.aborted, true);
assert.equal(watchdog.signal.reason, cancellation);
});

function assertPhaseDetails(reason: unknown, phase: string, timeoutMs: number): void {
assert.ok(reason instanceof AppError, `phase expiry must be an AppError, got ${String(reason)}`);
assert.equal(isTransferPhaseTimeoutError(reason), true);
Expand Down
12 changes: 11 additions & 1 deletion packages/host-kit/src/internal/transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ export type TransferPhaseWatchdog = {
signal: AbortSignal;
arm(phase: TransferPhase, timeoutMs: number): void;
touchBody(): void;
/** Stops counting an idle ceiling while the holder waits on demand, keeping its armed phase. */
suspend(): void;
/** Stops observing this attempt. A suspended phase stays suspended. */
dispose(): void;
};

Expand All @@ -168,11 +171,12 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe
const signal = callerSignal ? AbortSignal.any([callerSignal, phase.signal]) : phase.signal;
let armed: { phase: TransferPhase; timeoutMs: number } | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
let suspended = false;
const rearm = () => {
if (timer !== undefined) clearTimeout(timer);
timer = undefined;
const current = armed;
if (!current) return;
if (!current || suspended) return;
timer = setTimeout(() => {
timer = undefined;
phase.abort(createTransferPhaseTimeoutError(current.phase, current.timeoutMs));
Expand All @@ -183,13 +187,19 @@ export function createTransferPhaseWatchdog(callerSignal?: AbortSignal): Transfe
signal,
arm(nextPhase, timeoutMs) {
if (signal.aborted) return;
suspended = false;
armed = { phase: nextPhase, timeoutMs };
rearm();
},
touchBody() {
if (!armed || !BODY_PHASES.has(armed.phase)) return;
rearm();
},
suspend() {
suspended = true;
if (timer !== undefined) clearTimeout(timer);
timer = undefined;
},
dispose() {
armed = undefined;
if (timer !== undefined) clearTimeout(timer);
Expand Down
3 changes: 3 additions & 0 deletions packages/host-kit/src/transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ export {
readNodeHttpRequestBody,
readNodeHttpResponseBody,
timingSafeStringEqual,
TRANSFER_BODY_INACTIVITY_TIMEOUT_MS,
TRANSFER_CONNECT_TIMEOUT_MS,
TRANSFER_RESPONSE_HEADER_TIMEOUT_MS,
type TransferPhaseWatchdog,
type NodeHttpRequester,
} from './internal/transport.ts';
57 changes: 53 additions & 4 deletions packages/proxy/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,49 @@ Your Node HTTP client must send exactly one valid `Host` header on every request
configured. Configure raw-socket and HTTP/1.0 health probes to send `Host: localhost` (or your
deployment's host and port).

### How the proxy reaches the daemon

On artifact transfer routes — `POST /upload`, `/upload/preflight` and `/upload/finalize`,
`PUT /upload/direct/*` and `GET /artifacts*` — the proxy forwards with `node:http` / `node:https`.
Each transfer has a connect limit of 30 seconds, a response-header wait of 30 seconds that
starts only once its own outgoing body has finished and the connection is established, and a
120-second limit on body inactivity. For downloads, that limit counts time spent waiting for
upstream bytes while you read the response; it pauses when you stop reading. Every forwarded
exchange also ends at 600 seconds regardless of how steadily its bytes move. Your `upstreamTimeoutMs`
is an additional cap. The earliest caller cancellation, operator cap, phase limit or aggregate expiry
ends the exchange. A transfer does not retry or restart its aggregate.

Transfer phase and aggregate timeout error responses use HTTP 500 and include `reason`,
`transferPhase` and `timeoutMs`. These fields appear only for `COMMAND_FAILED` with
`reason: transfer_phase_timeout` and the matching connect/header (30000ms) or body (120000ms)
phase budget, or `reason: transfer_forwarding_deadline_exceeded` with
`transferPhase: forwarding` and `timeoutMs: 600000`. Other errors omit these fields.
After the proxy returns a response, a timeout while you consume it fails the body read instead of
replacing its status. Preflight bodies are consumed before ticket rewriting; a timeout there returns
HTTP 500.

On these routes:

- Redirects are returned without being followed. The daemon's `308` resume response keeps its
`Range` and `x-upload-offset` headers.
- Default request headers added by `fetch`, such as `accept` and `user-agent`, are not sent.
- A patched global `fetch` or an `undici` global dispatcher no longer intercepts the request.

Supply `upstreamFetch` if you need those Fetch behaviors or your own routing.

Environment-proxy routing depends on your runtime. In loopback checks, Node 22.12 ignores
`HTTP_PROXY` and `HTTPS_PROXY` for both transports. Node 26.11.1 uses them when
`NODE_USE_ENV_PROXY=1`. Bun 1.4.2 Fetch uses them without that opt-in, while the default transfer
transport requires it. On Node 26.11.1 and Bun 1.4.2, `NO_PROXY=127.0.0.1` bypasses the proxy.
If your Bun deployment needs proxy routing for transfers, set `NODE_USE_ENV_PROXY=1` or supply
`upstreamFetch`.

By default, `/rpc`, `/health` and request diagnostics use global `fetch`.

### Reach the daemon over your own transport

By default the proxy calls the daemon with the global `fetch`. Pass `upstreamFetch` when the daemon
is only reachable some other way, such as a tunnel the device host opened to your gateway:
Pass `upstreamFetch` when the daemon is only reachable some other way, such as a tunnel the device
host opened to your gateway:

```ts
const proxy = createDaemonProxy({
Expand All @@ -111,6 +150,16 @@ const proxy = createDaemonProxy({
`upstreamFetch` receives a `Request` addressed to `upstreamBaseUrl` and must resolve with the
daemon's `Response`. Abort the exchange when `request.signal` aborts.

On transfer routes, the proxy bounds its wait for your transport and response body. It settles on
your abort, your `upstreamTimeoutMs`, its 600-second aggregate and its demand-aware response-body
inactivity limit. Work your transport starts can keep running if it ignores `request.signal`. The
proxy does not wait for that work or for a cancellation callback that never settles. It cannot
measure a connect or a header wait it does not own, so on transfer routes your transport keeps those
obligations. Bound connection establishment to 30 seconds and the response-header wait to 30 seconds
once the connection is established and the outgoing body has finished. Stream `request.body` with
120-second body inactivity limits. Stop your request and response sources on cancellation. A body
pull alone does not confirm that an outgoing byte was sent.

### Authenticate each client yourself

One `clientToken` is shared by every client. To give each client its own credential, pass
Expand Down Expand Up @@ -158,9 +207,9 @@ authenticate: async (request) => {
| `upstreamToken` | required | The daemon's auth token. |
| `clientToken` | required without `authenticate` | Token clients must send as their daemon auth token. |
| `authenticate` | none | Authenticates each client and names its tenant; replaces `clientToken`. |
| `upstreamFetch` | global `fetch` | Sends one request to the daemon. |
| `upstreamFetch` | none | Sends one request to the daemon. Without it, transfer routes use `node:http(s)` and every other route uses the global `fetch`. |
| `maxRpcBodyBytes` | 64 MiB, the daemon's own request limit | Larger `/rpc` bodies get a 400 response. |
| `upstreamTimeoutMs` | none | Longest time one daemon request may take, including its body. Without it, a request lasts as long as the client waits, and the client's command timeout bounds it. |
| `upstreamTimeoutMs` | none | Independent cap on one daemon request, including its body. `/health` keeps its separate five-second limit. Without this option, other requests last as long as the client waits, bounded by the client's command timeout and, on transfer routes, the 600-second forwarding aggregate. |

`maxRpcBodyBytes` limits the bytes of one `/rpc` body. The proxy reads the whole body before it
authenticates the client, then decodes and parses it. One unauthenticated request can therefore
Expand Down
Loading
Loading