From b34df43734e2e474744b9626fd12b3ec20eb52c3 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 07:12:34 +0900 Subject: [PATCH 1/3] fix(link): report stale child connection readiness --- .../261107_orphaned_child_link/010_design.md | 39 +++++++++++++++++++ .../src/content/docs/guides/remote-link.md | 4 +- src/client/machine-listener.ts | 13 +++++++ structure/runtime.md | 2 +- tests/clients/client-machine-listener.test.ts | 28 +++++++++++++ tests/clients/link-readiness.test.ts | 1 + 6 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 devlog/_plan/261107_orphaned_child_link/010_design.md diff --git a/devlog/_plan/261107_orphaned_child_link/010_design.md b/devlog/_plan/261107_orphaned_child_link/010_design.md new file mode 100644 index 00000000000..f2db889e312 --- /dev/null +++ b/devlog/_plan/261107_orphaned_child_link/010_design.md @@ -0,0 +1,39 @@ +# Orphaned Remote Link Child (#8243) + +## Failure and hypotheses + +`src/client/runtime.ts` binds a startup connection snapshot; `src/client/machine-listener.ts` +reports tunnel readiness without checking that snapshot against the durable connection. Meanwhile, +`src/client/connect.ts` refuses disconnect after the record disappears unless a disconnect receipt +already reached the clearing phase. Pinned sync in `src/client/sync-expectation.ts` correctly refuses +the missing connection and must keep doing so without writes. + +Possible causes of the missing record include an old whole-config writer, a concurrent config +mutation, or an external replacement. An audit of config persistence and regression fixtures can +exclude or demonstrate individual cooperating writers; it cannot identify the historical external +writer without evidence from the affected machine. The runtime symptom is falsified if readiness +changes when only durable ownership changes; the disconnect symptom is falsified if the existing +locked transaction can complete from a missing record and safely identified artifacts. + +## Design and verification + +Use durable connection identity as the authority for listener readiness and status. A missing, +invalid, or different owner/link is distinct from a tunnel failure. Do not derive a connection +from health, tunnel state, or catalog. Revalidate under the existing lifecycle and config locks +before any cleanup; preserve unknown or replaced credentials, sidecars, Desktop state, and Codex +state. Only remove artifacts with independently established ownership, and make interruption and +concurrent reconnect safe. Preserve pinned sync's exit 5 and no-write contract. + +Add failing-first focused tests for the stale listener, orphan cleanup, replacement races and +negative ownership cases. Audit whole-config writers and protect a proven cooperating overwrite +where feasible. Update the owning structure and user-facing docs. Run focused tests, typecheck, +`test:changed`, structure and privacy gates, and the docs build; document unrun coverage. + +## Open recovery question + +The existing `client-link.json` carries a link ID and tunnel details, but no token fingerprint. +The token file and this sidecar can each have been replaced independently. An absent connection +therefore does not establish that the present token belongs to the sidecar, and a token read +followed by compare-and-swap alone only proves that the token did not change during cleanup. A +recovery implementation needs an independent owner witness and a safe path for legacy orphans; +it must not interpret sidecar presence as proof of token ownership. diff --git a/docs-site/src/content/docs/guides/remote-link.md b/docs-site/src/content/docs/guides/remote-link.md index 5fd0d404030..6362f1c64c8 100644 --- a/docs-site/src/content/docs/guides/remote-link.md +++ b/docs-site/src/content/docs/guides/remote-link.md @@ -69,7 +69,9 @@ A failed link does not silently switch to a local provider. The Child returns a retryable `503` without forwarding the link key or request when its tunnel is failed, stopped, or not supervised. It also keeps the key local when another process owns the tunnel port; free that port so the Child's SSH tunnel can bind it. -On a Child, `GET /healthz` only says that OpenCodex is running. `GET /readyz` (and `ocx ready`) also says whether the link can carry a request right now: while it cannot, it answers `503` with `Retry-After: 1`, `"status": "pending"` and a `link` object such as `{ "state": "unavailable", "reason": "home_unreachable" }`, and while it can, `200` with `"status": "ready"` and `{ "state": "available" }`. The reasons are `tunnel_connecting`, `tunnel_failed`, `tunnel_idle`, `tunnel_unavailable`, `sidecar_invalid`, `home_unreachable`, `home_unauthorized`, `link_auth_failed`, `link_credential_unavailable` and `link_unverified`. A Child that joined from its own dashboard answers from its tunnel. A Child that its Home added has no tunnel of its own to look at, so it asks the Home through the tunnel with the same credential-free check every relayed request starts with; it repeats that check at most once every five seconds, and answers `link_unverified` when the Home has not replied within half a second. +On a Child, `GET /healthz` only says that OpenCodex is running. `GET /readyz` (and `ocx ready`) also says whether the link can carry a request right now: while it cannot, it answers `503` with `Retry-After: 1`, `"status": "pending"` and a `link` object such as `{ "state": "unavailable", "reason": "home_unreachable" }`, and while it can, `200` with `"status": "ready"` and `{ "state": "available" }`. The reasons are `tunnel_connecting`, `tunnel_failed`, `tunnel_idle`, `tunnel_unavailable`, `sidecar_invalid`, `home_unreachable`, `home_unauthorized`, `link_auth_failed`, `link_credential_unavailable`, `link_unverified` and `client_state_changed`. A Child that joined from its own dashboard answers from its tunnel. A Child that its Home added has no tunnel of its own to look at, so it asks the Home through the tunnel with the same credential-free check every relayed request starts with; it repeats that check at most once every five seconds, and answers `link_unverified` when the Home has not replied within half a second. + +The reason `client_state_changed` means the running Child's saved connection is missing or now names another owner or link. Its machine and link status endpoints also return `503` with this reason. Stop or restart the Child and check its connection before re-linking; this readiness result does not itself disconnect the relay. To confirm which Home is serving a Child, compare the `x-opencodex-served-by` header on the Child's `GET /v1/models` with the `instanceId` in the Home's `GET /healthz`. The Child passes the Home's header through unchanged, so the two match while the link works. The value is an opaque id, never a hostname; see [Home identity and the served-by header](/reference/management-api/#home-identity-and-the-served-by-header). diff --git a/src/client/machine-listener.ts b/src/client/machine-listener.ts index 554d0488554..61c1eaa0007 100644 --- a/src/client/machine-listener.ts +++ b/src/client/machine-listener.ts @@ -41,6 +41,7 @@ import { linkRouteAllowed } from "../link/routes"; import { linkHomeName, linkHomePhrase } from "./link-home"; import { relayLinkManagementRead } from "./link-management-relay"; import type { ClientRestartAdmission } from "./runtime-restart"; +import { clientRestartRollbackAction } from "./runtime-restart"; const VERSION = packageVersion("0.0.0"); const GUI_SPA_PATHS = new Set([ @@ -133,6 +134,7 @@ export function startMachineListener( return state.value; })(); const linkMode = isLinkConnection(connection); + const connectionChanged = () => clientRestartRollbackAction(connection, readClientConnectionState()) === "recycle"; if (linkMode && !connection.link) throw new Error("link machine listener requires link transport metadata"); const managementAuth = deps.managementAuthState ?? initializeManagementAuthState(config); let hubReachability: HubReachability = "unknown"; @@ -196,9 +198,17 @@ export function startMachineListener( const identity = { service: "opencodex", version: VERSION, role: "client" } as const; const facts = { uptime: process.uptime(), pid: process.pid, port: server.port, protocolVersion: 1 }; let link: LinkReadiness | null = null; + if (linkMode && connectionChanged()) { + return Response.json({ ...identity, status: "pending", ...facts, link: { state: "unavailable", reason: "client_state_changed" } }, + { status: 503, headers: { "Retry-After": "1", "Cache-Control": "no-store" } }); + } if (linkMode && deps.linkReadiness) { try { link = await deps.linkReadiness(); } catch { link = { ready: false, reason: "link_unverified" }; } } + if (linkMode && connectionChanged()) { + return Response.json({ ...identity, status: "pending", ...facts, link: { state: "unavailable", reason: "client_state_changed" } }, + { status: 503, headers: { "Retry-After": "1", "Cache-Control": "no-store" } }); + } if (link && !link.ready) { // The /readyz contract: not ready is 503 with Retry-After and status pending or failed. return Response.json({ ...identity, status: "pending", ...facts, link: { state: "unavailable", reason: link.reason } }, @@ -247,6 +257,9 @@ export function startMachineListener( if (managementPrincipal(req, managementAuth, config) !== "gui-session") { return Response.json({ error: "opencodex machine GUI session required" }, { status: 401 }); } + if (linkMode && connectionChanged() && (url.pathname === "/api/machine/status" || url.pathname === "/api/link/status")) { + return Response.json({ error: "client_state_changed" }, { status: 503, headers: { "Cache-Control": "no-store" } }); + } // A loopback dashboard session proves possession, not user presence: any local // process can fetch the dashboard bootstrap and replay its token and CSRF value. // Keep the connected listener useful for status/diagnostics, but never let that diff --git a/structure/runtime.md b/structure/runtime.md index 69413c657a4..273ab85707d 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -180,7 +180,7 @@ until shutdown. Standalone shutdown restores native Codex; connected hub clients uninstall still restore. `src/service/cli.ts` removes the service token on uninstall only when persisted client state is disconnected and no pending connect marker owns the newly issued key. `src/client/connect.ts` publishes that fingerprint marker before the key, then clears it with the connection commit or rollback under the client lifecycle and config mutation locks. Connected, invalid, or mismatched client state retains an existing token. A valid pending marker retains only its matching fingerprint; an older marker does not own a replacement service key. An absent token is reported as absent; unsafe, malformed, or unreadable markers and lock, state-read, or deletion failures leave cleanup unverified. `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. -`ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. +`ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state before and after readiness probing. If they differ, `/readyz` and authenticated machine/link status answer 503 with `client_state_changed`. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks diff --git a/tests/clients/client-machine-listener.test.ts b/tests/clients/client-machine-listener.test.ts index 3ef6a52179c..fdff22ddd4e 100644 --- a/tests/clients/client-machine-listener.test.ts +++ b/tests/clients/client-machine-listener.test.ts @@ -353,6 +353,31 @@ describe("client machine listener in link mode", () => { }); const writeKey = () => writeFileSync(join(root, "service-api-token"), `${LINK_KEY}\n`, { mode: 0o600 }); + test("readiness reports durable connection divergence even when the tunnel is ready", async () => { + const state = linkConnection(); + const save = (client: unknown, runtimeRole: string | undefined = "client") => { + writeFileSync(join(root, "config.json"), JSON.stringify({ port: 0, providers: {}, runtimeRole, client })); + }; + save(state); + const server = startMachineListener(0, { + state, managementAuthState: authState(), linkReadiness: () => ({ ready: true }), + }); + servers.push(server); + expect((await fetch(new URL("/readyz", server.url))).status).toBe(200); + const headers = await guiHeaders(server); + expect((await fetch(new URL("/api/machine/status", server.url), { headers })).status).toBe(200); + for (const next of [undefined, { ...state, link: { ...state.link!, linkId: `lnk_${"c".repeat(16)}` } }, + { ...state, connectedAt: "2026-09-27T00:00:00.000Z" }]) { + save(next, next ? "client" : undefined); + const response = await fetch(new URL("/readyz", server.url)); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ status: "pending", link: { state: "unavailable", reason: "client_state_changed" } }); + const status = await fetch(new URL("/api/machine/status", server.url), { headers }); + expect(status.status).toBe(503); + expect(await status.json()).toEqual({ error: "client_state_changed" }); + } + }); + function linkListener(options: { fingerprint?: string; linkStatus?: () => ClientLinkSupervisorStatus; @@ -360,6 +385,9 @@ describe("client machine listener in link mode", () => { reply?: () => Response; } = {}) { const upstream: Request[] = []; + writeFileSync(join(root, "config.json"), JSON.stringify({ + port: 0, providers: {}, runtimeRole: "client", client: linkConnection(options.fingerprint), + })); const server = startMachineListener(0, { state: linkConnection(options.fingerprint), managementAuthState: authState(), diff --git a/tests/clients/link-readiness.test.ts b/tests/clients/link-readiness.test.ts index 6fad9f3c2aa..cfec683cdd6 100644 --- a/tests/clients/link-readiness.test.ts +++ b/tests/clients/link-readiness.test.ts @@ -176,6 +176,7 @@ function linkConnection(): OcxClientConnectionConfig { test("a Child's /readyz reports an unusable link as 503 pending with a reason, and /healthz stays live", async () => { let verdict: LinkReadiness = { ready: false, reason: "home_unreachable" }; + writeFileSync(join(root, "config.json"), JSON.stringify({ port: 0, providers: {}, runtimeRole: "client", client: linkConnection() })); const server = startMachineListener(0, { state: linkConnection(), managementAuthState: authState(), linkKeySource: () => LINK_KEY, linkTunnel: { connected: () => ({ kind: "match" }), pending: () => false, waitForConnected: async () => false }, From 0c83402251b970898cdb78c62cf21f045a2b0e05 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 07:36:10 +0900 Subject: [PATCH 2/3] test(link): seed durable state for tunnel readiness --- tests/clients/link-readiness.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/clients/link-readiness.test.ts b/tests/clients/link-readiness.test.ts index bcbe9a11bb6..aa8f2f6bc3e 100644 --- a/tests/clients/link-readiness.test.ts +++ b/tests/clients/link-readiness.test.ts @@ -204,6 +204,7 @@ test("a Child's /readyz reports an unusable link as 503 pending with a reason, a test("a ready Child's /readyz names its current tunnel connection, and a pending or unknown one omits it", async () => { let verdict: LinkReadiness = { ready: false, reason: "home_unreachable" }; let generation: string | undefined = "fixture-child:1"; + writeFileSync(join(root, "config.json"), JSON.stringify({ port: 0, providers: {}, runtimeRole: "client", client: linkConnection() })); const server = startMachineListener(0, { state: linkConnection(), managementAuthState: authState(), linkKeySource: () => LINK_KEY, linkTunnel: { connected: () => ({ kind: "match" }), pending: () => false, waitForConnected: async () => false }, From c6d8ced4027974b8c25e3f1da20d3f85e2cfece3 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 07:45:40 +0900 Subject: [PATCH 3/3] fix(link): fence stale client status reads --- src/client/machine-listener.ts | 2 +- structure/remote-link.md | 4 +-- tests/clients/client-machine-listener.test.ts | 30 +++++++++++++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/client/machine-listener.ts b/src/client/machine-listener.ts index 9c94a45a4c4..3e2f2c7781a 100644 --- a/src/client/machine-listener.ts +++ b/src/client/machine-listener.ts @@ -263,7 +263,7 @@ export function startMachineListener( if (managementPrincipal(req, managementAuth, config) !== "gui-session") { return Response.json({ error: "opencodex machine GUI session required" }, { status: 401 }); } - if (linkMode && connectionChanged() && (url.pathname === "/api/machine/status" || url.pathname === "/api/link/status")) { + if (linkMode && connectionChanged() && (url.pathname === "/api/machine/status" || url.pathname === "/api/machine/clients" || url.pathname === "/api/link/status")) { return Response.json({ error: "client_state_changed" }, { status: 503, headers: { "Cache-Control": "no-store" } }); } // A loopback dashboard session proves possession, not user presence: any local diff --git a/structure/remote-link.md b/structure/remote-link.md index 75dbd235f5a..23b4a6157a7 100644 --- a/structure/remote-link.md +++ b/structure/remote-link.md @@ -54,7 +54,7 @@ For an empty-store Home-initiated Child, `src/link/status-projection.ts` derives The dashboard link routes (`GET /api/link/status`, `GET /api/link/candidates`, `POST /api/link/probe`, `POST /api/link/confirm-host`, `POST /api/link/apply`, `POST /api/link/{id}/reconnect` and `DELETE /api/link/{id}`) admit a paired GUI session, or, on a standalone runtime only, the current loopback-issued GUI session that reached the public listener bound to a loopback hostname. `POST /api/link/join` is stricter and requires the paired session because it durably redirects this machine's client traffic. The hub-link, hub-management and claude-intercept ingresses are never trusted loopback ingress, a stale session is refused, and a hub keeps the paired-only rule. The Tailscale identity refusal runs before either check. A loopback session is minted by the loopback dashboard bootstrap without a credential, so it proves possession, not user presence: any local process can fetch the bootstrap and replay its token and CSRF value, which is why it cannot authorize joining and why `src/client/machine-listener.ts` keeps durable machine changes on the connected listener away from that session. The remaining link routes accept this casual-path trade, the same as `POST /api/github/star` in `src/server/management/sidebar-routes.ts`; it is not a secret-backed boundary like an operator-created pairing grant or the admin token. -A successful join restarts this proxy (a 503 drain of up to a minute while running turns finish, then a closed listener) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s drain plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. +A successful join restarts this proxy (a 503 drain of up to a minute while running turns finish, then a closed listener) into the client runtime on the configured port. `GET /api/link/status` tells a GUI-session caller whether it may join: the response gains `joinAvailable`, true only for a paired dashboard session on a standalone runtime whose live port is its configured port, and `joinDenied`, which is `null` when allowed or the first failed gate (`pairing_required`, `standalone_required`, then `join_port_mismatch`). On a Child whose durable connection diverges from its startup snapshot, link and machine status instead return `503 client_state_changed`, as does readiness; this reports the divergence without stopping or reconstructing the relay. An admin-token caller gets the exact K16 document without either field, because `ocx link status` validates it key by key. The dashboard reads an absent availability field as false and an absent or unknown denial as generic unavailability; while unavailable the Child role card cannot be selected by pointer or keyboard. A pairing refusal says to pair this machine first and offers code entry only on the existing same-origin literal-loopback transport. A port refusal instead asks the operator to restart on the configured port; neither status reads nor displaying this guidance mint a grant or change pairing, keys, routing, or runtime state. Before **Connect as Child** the confirmation panel says that the restart briefly interrupts Codex and that Codex keeps its local address. The dashboard reads the standalone's pid from same-origin `/healthz`, sends the join, then reads `/healthz` once a second, skipping status polls meanwhile, and reloads only when it reports `role: "client"` under another pid, so the reloaded document carries the client role and a fresh session. It never gives up by itself: past the server's own handoff budget (60 s drain plus 70 s replacement readiness, plus a margin: 145 seconds) it says the restart is slow and keeps reading `/healthz` every 5 seconds until the Child answers or the page is left. Manual Hub pairing recovery is offered only for a missing session or HTTP 401/403 from status, and moves keyboard focus into the code input. Session validation inherits the form abort signal; navigation or unmount prevents completion from updating App state. @@ -82,7 +82,7 @@ Codex keeps the standalone loopback routing: `routingTarget` in `src/client/conn `src/client/link-relay.ts` forwards exactly the `linkRouteAllowed` routes from `src/link/routes.ts` through the tunnel. It drops the caller's `Authorization`, Azure `api-key`, Anthropic-compatible `x-api-key`, Google `x-goog-api-key`, `x-opencodex-api-key`, `chatgpt-account-id` and `cookie` and sends the link key as `Authorization: Bearer`, the wire an `env_key` config sent; `GET /v1/usage` takes it as `x-opencodex-api-key`, the only header that route admits. The explicit provider forms matter because the Child and Home are separate credential owners: caller provider credentials never cross the tunnel merely because they are not Bearer tokens. The Home admits the link key and serves the Child with its own accounts. The request body is streamed chunk by chunk with the caller's `Content-Length` and a byte-counting cap at the inbound limit (`resolveInboundBodyLimitBytes`, 256 MiB by default); a larger declared or streamed body answers 413. A lone `Transfer-Encoding: chunked` without `Content-Length` is admitted as a standalone admits it, because the listener has already de-chunked the body; any other Transfer-Encoding, or one next to a `Content-Length`, answers 400. The Home's response headers may take up to 300 seconds, and a caller abort ends the wait sooner. SSE passes through chunk by chunk with caller-abort propagation and a 300-second idle limit, other response bodies stream under the same byte cap, and the relay answers 503 with Retry-After while the tunnel is down. The client supervisor is the relay's tunnel gate (`LinkTunnelGate`): only while the tunnel is connecting or reconnecting (including the start of the client runtime) does a relayed request wait, for at most 15 seconds (`LINK_RELAY_HOLD_MS`) from its first wait and with at most 64 requests waiting, before it is forwarded once; a connected tunnel costs one `pending()` call per request, and a failed one answers 503 at once. A forward whose connection was refused sent nothing, so while the tunnel reconnects it may wait again and be sent again inside the same 15 seconds, provided the streamed body was never read or cancelled; any other failure (a reset, a timeout, a failure after the body started) is never replayed. Both the Child's machine listener and the Home's hub-link listener bind with `idleTimeout: 255`, the public listener's limit, so a held or slow turn is not cut by Bun's 10-second default. Like a standalone data route, a relayed request then lifts its own idle timer (`server.timeout(req, 0)` in `src/client/link-ingress.ts`), so a quiet stretch longer than 255 seconds inside a long generation is not cut either; the relay's header deadline, SSE idle limit and caller abort bound the wait instead. Hub transport keeps the 4 MiB management-relay listener bound and its default idle limit. Link mode waits for the configured port without signalling its holder, then binds there or fails; `src/client/runtime.ts` passes the cached link key, tunnel status and tunnel gate through `bindClientListener` to every bind attempt. Link mode replaces the hub management relay with the read-only link management relay (see [Child management relay and Home Codex settings](#child-management-relay-and-home-codex-settings)) and refuses key rotation and revocation, which belong to the Home. -`src/client/link-readiness.ts` gives a link-mode Child's `GET /readyz` its verdict; `/healthz` keeps meaning only that the process answers. While relayed requests cannot be served, `/readyz` answers `503` with `Retry-After: 1`, `status: "pending"` and `link: { state: "unavailable", reason }`, which the strict readiness contract in `src/server/proxy-liveness.ts` reads as ours and not ready; otherwise `200`, `status: "ready"` and `link: { state: "available" }`. A reason is one fixed word. A Child-initiated link reads its supervisor: connected is ready unless the keyed probe last saw `home_unreachable` or `unauthorized` (`home_not_ready` stays ready, because relayed requests do not depend on the Home's own startup readiness); connecting, reconnecting and a failed state with a retry in flight read `tunnel_connecting`, failed `tunnel_failed`, idle or stopped `tunnel_idle`, an unreadable sidecar `sidecar_invalid`. A Home-initiated link has no local tunnel to inspect, so it runs `probeLinkRelayPreflight` from `src/client/link-relay-transport.ts`: the credential-free challenge every relayed request starts with, on its own connection, without a link key, data request or body, and without reading the response body. It sends `x-opencodex-link-probe: 1` (`LINK_RELAY_PROBE_HEADER`), so the Home verifies and answers without reserving a session (see the reservation book below). A verified proof is ready, no answer within the five-second authentication deadline is `home_unreachable`, any other answer `link_auth_failed`, and a missing link key `link_credential_unavailable` with nothing sent. One preflight runs at a time, its verdict is reused for five seconds, and `/readyz` waits at most 500 ms for a running one before it answers `link_unverified`, so an unauthenticated local caller causes at most one preflight every five seconds. A link with neither a supervisor nor a Home-initiated gate reads `tunnel_unavailable`. Restart handoff and the update job judge a replacement by `/healthz`, so a Child whose Home is down still restarts. +`src/client/link-readiness.ts` gives a link-mode Child's `GET /readyz` its verdict; `/healthz` keeps meaning only that the process answers. While relayed requests cannot be served, or the saved client owner or link differs from the runtime snapshot, `/readyz` answers `503` with `Retry-After: 1`, `status: "pending"` and `link: { state: "unavailable", reason }`, which the strict readiness contract in `src/server/proxy-liveness.ts` reads as ours and not ready; otherwise `200`, `status: "ready"` and `link: { state: "available" }`. A reason is one fixed word. A Child-initiated link reads its supervisor: connected is ready unless the keyed probe last saw `home_unreachable` or `unauthorized` (`home_not_ready` stays ready, because relayed requests do not depend on the Home's own startup readiness); connecting, reconnecting and a failed state with a retry in flight read `tunnel_connecting`, failed `tunnel_failed`, idle or stopped `tunnel_idle`, an unreadable sidecar `sidecar_invalid`. A Home-initiated link has no local tunnel to inspect, so it runs `probeLinkRelayPreflight` from `src/client/link-relay-transport.ts`: the credential-free challenge every relayed request starts with, on its own connection, without a link key, data request or body, and without reading the response body. It sends `x-opencodex-link-probe: 1` (`LINK_RELAY_PROBE_HEADER`), so the Home verifies and answers without reserving a session (see the reservation book below). A verified proof is ready, no answer within the five-second authentication deadline is `home_unreachable`, any other answer `link_auth_failed`, and a missing link key `link_credential_unavailable` with nothing sent. One preflight runs at a time, its verdict is reused for five seconds, and `/readyz` waits at most 500 ms for a running one before it answers `link_unverified`, so an unauthenticated local caller causes at most one preflight every five seconds. A link with neither a supervisor nor a Home-initiated gate reads `tunnel_unavailable`. Restart handoff and the update job judge a replacement by `/healthz`, so a Child whose Home is down still restarts. > Decision record: [ADR-6032](decisions/ADR-6032-link-relay-credential-boundary.md) diff --git a/tests/clients/client-machine-listener.test.ts b/tests/clients/client-machine-listener.test.ts index fdff22ddd4e..a6a2e214c0d 100644 --- a/tests/clients/client-machine-listener.test.ts +++ b/tests/clients/client-machine-listener.test.ts @@ -372,12 +372,36 @@ describe("client machine listener in link mode", () => { const response = await fetch(new URL("/readyz", server.url)); expect(response.status).toBe(503); expect(await response.json()).toMatchObject({ status: "pending", link: { state: "unavailable", reason: "client_state_changed" } }); - const status = await fetch(new URL("/api/machine/status", server.url), { headers }); - expect(status.status).toBe(503); - expect(await status.json()).toEqual({ error: "client_state_changed" }); + for (const path of ["/api/machine/status", "/api/machine/clients", "/api/link/status"]) { + const status = await fetch(new URL(path, server.url), { headers }); + expect(status.status).toBe(503); + expect(await status.json()).toEqual({ error: "client_state_changed" }); + } } }); + test("readiness rechecks durable ownership after an in-flight tunnel probe", async () => { + const state = linkConnection(); + writeFileSync(join(root, "config.json"), JSON.stringify({ port: 0, providers: {}, runtimeRole: "client", client: state })); + let resolveProbe!: () => void; + let notifyProbe!: () => void; + const started = new Promise(resolve => { notifyProbe = resolve; }); + const pending = new Promise(resolve => { resolveProbe = resolve; }); + const server = startMachineListener(0, { + state, managementAuthState: authState(), linkReadiness: async () => { notifyProbe(); await pending; return { ready: true }; }, + }); + servers.push(server); + try { + const responsePromise = fetch(new URL("/readyz", server.url)); + await started; + writeFileSync(join(root, "config.json"), JSON.stringify({ port: 0, providers: {} })); + resolveProbe(); + const response = await responsePromise; + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ link: { state: "unavailable", reason: "client_state_changed" } }); + } finally { resolveProbe(); } + }); + function linkListener(options: { fingerprint?: string; linkStatus?: () => ClientLinkSupervisorStatus;