diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index dc6c9e6e7fd..cee0ea5a84d 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -745,6 +745,27 @@ both named volumes and a synthetic catalog survive replacement. This check does real provider account, OAuth callback, custom mount migration, or every CPU architecture; perform the authenticated routed-response check above for your deployment. +### Restarts after an unclean exit + +A container runs the proxy as PID 1 on every start. After an unclean exit (an OOM kill, a crash, a +node failure) the state volume can still hold `ocx.pid` and `runtime-port.json` naming PID 1, and +the next start has to decide whether that owner is gone. It recovers on its own when the record was +written by the same container in an earlier run, by an OpenCodex version that records where its +owner runs: a Kubernetes container restarted inside its pod while the node stays up, or the same +Docker or Podman container started again (`docker restart`, a restart policy, including after a +host reboot) when its writable layer has a random id, as with Docker's `overlay2` storage driver +and Podman. Docker's containerd image store numbers its layers instead, so a restart there refuses. + +Every other case refuses with `Proxy ownership is unverified; startup refused`, because the old +owner could still be serving the same volume from another container: a replacement Kubernetes pod, +a recreated Compose container, a pod that comes back after a node reboot, another node, or a record +written by an older version (the first restart after an upgrade). Confirm that nothing else serves this state directory, then move +`runtime-port.json` and `ocx.pid` aside once and start again. Run a single writer per state +volume: a Kubernetes `Deployment` that shares a volume should use `strategy: Recreate` (or +`maxSurge: 0`) so the old pod stops before the new one starts. Do not share one state volume between +processes in one network namespace (host networking, sidecars, several WSL distros), between +sandboxes restored from one VM snapshot, or between machines with the same host name. + ## Rollback Inspect existing Serve mappings before changing them. `tailscale serve reset` removes every mapping diff --git a/scripts/test-layout/layout-start-ownership.json b/scripts/test-layout/layout-start-ownership.json index 7caaff72af5..c8e16c53002 100644 --- a/scripts/test-layout/layout-start-ownership.json +++ b/scripts/test-layout/layout-start-ownership.json @@ -4,6 +4,7 @@ "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-self-pid-record.test.ts": "server", "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server" } diff --git a/src/config/process-state.ts b/src/config/process-state.ts index b8732cc50fc..a80bdefa035 100644 --- a/src/config/process-state.ts +++ b/src/config/process-state.ts @@ -1,5 +1,7 @@ import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, readlinkSync, unlinkSync } from "node:fs"; +import { hostname } from "node:os"; import { join } from "node:path"; import { isLocalAttestationSecret } from "../lib/local-management-attestation"; import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; @@ -27,8 +29,122 @@ function ensureProcessStateDir(): void { else hardenConfigDir(); } +/** + * Whether this process has written `ocx.pid` or `runtime-port.json` in its own lifetime (#8552). + * + * Set before the write, deliberately even when the write then throws, and never cleared: a record + * this process wrote (or may have written) and later removed still means its PID once named a live + * owner of this home, so it can never be told apart from an earlier lifetime of the same PID number. + * Moving the assignment after the write would let a failed write fail open. + */ +let publishedOwnershipRecords = false; + +export function hasPublishedOwnershipRecords(): boolean { + return publishedOwnershipRecords; +} + +/** + * Where the owner of a runtime record ran, as digests that store none of their inputs (#8552). + * + * Every container runs the proxy as PID 1, so a PID alone cannot say whether a record came from + * this container's previous run or from another container serving the same state volume. The + * `host` name must match, and on Linux at least one of two stronger facts as well + * ({@link sameRuntimeOwnerScope}): + * + * - `network` with `boot`, the network namespace within one boot: a container restarted inside + * its Kubernetes pod keeps it, because the pod sandbox owns it. Namespace inode numbers repeat + * across machines and boots, so the boot id has to match too; + * - `container`, the writable layer of the overlay mounted at `/` (its `upperdir`), counted only + * when its path carries a random 64-hex layer id (Docker's `overlay2` driver, containers/storage + * for Podman and CRI-O): a container started again keeps it, across a host reboot as well, and + * no other container ever mounts it, not even one sharing its network (`--network container:`) + * or its pod. A layer named by a sequence number (containerd's overlayfs snapshotter, which + * nodes built alike hand out identically) or a host whose own root is an overlay proves nothing + * beyond one machine, so it is not used; nor is a container id, which a container joined to + * another's network sees in mountinfo. + * + * A replacement pod, a second container with a fixed `hostname`, or another node matches neither. + * Processes that share one network namespace (host networking, pod sidecars, WSL distros, which + * share one VM) can match on `network`; for them the loopback probe still finds a listening owner. + * Elsewhere the PID space is the whole machine and the host name suffices. Linux evidence that + * cannot be read yields a per-process value that matches no other lifetime, so the self-record + * proof fails closed. + */ +export interface RuntimeOwnerScope { + host: string; + boot?: string; + network?: string; + container?: string; +} + +let cachedRuntimeOwnerScope: RuntimeOwnerScope | undefined; + +export function runtimeOwnerScope(): RuntimeOwnerScope { + cachedRuntimeOwnerScope ??= computeRuntimeOwnerScope(); + return cachedRuntimeOwnerScope; +} + +function scopeDigest(...parts: string[]): string { + return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32); +} + +/** + * The `upperdir` of the overlay mounted at `/` when it names a randomly identified writable layer + * (a `/<64 hex>/` path segment): a container's own layer, unique across machines. The last root + * mount wins, as it is the one visible. Any other root filesystem (bare host, btrfs/zfs/vfs drivers, + * fuse-overlayfs, an overlay host root, containerd's sequence-numbered snapshots) yields nothing. + */ +export function rootOverlayUpperdirFromMountinfo(mountinfo: string): string | undefined { + let upperdir: string | undefined; + for (const line of mountinfo.split("\n")) { + const separator = line.indexOf(" - "); + if (separator < 0) continue; + const fields = line.slice(0, separator).split(" "); + if (fields[4] !== "/") continue; + const [fstype, , superOptions = ""] = line.slice(separator + 3).split(" "); + const match = fstype === "overlay" ? /(?:^|,)upperdir=([^,]+)/.exec(superOptions) : null; + upperdir = match && /\/[0-9a-f]{64}\//.test(match[1]) ? match[1] : undefined; + } + return upperdir; +} + +function readOptional(read: () => string): string | undefined { + try { return read(); } catch { return undefined; } +} + +function computeRuntimeOwnerScope(): RuntimeOwnerScope { + if (process.platform !== "linux") return { host: scopeDigest(process.platform, hostname()) }; + const boot = readOptional(() => readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()); + const network = readOptional(() => readlinkSync("/proc/self/ns/net")); + const container = readOptional(() => rootOverlayUpperdirFromMountinfo(readFileSync("/proc/self/mountinfo", "utf8")) ?? ""); + const scope: RuntimeOwnerScope = { host: scopeDigest("linux", hostname()) }; + if (boot && /^[0-9a-f-]{36}$/.test(boot) && network && /^net:\[\d+\]$/.test(network)) { + scope.boot = scopeDigest("boot", boot); + scope.network = scopeDigest("net", network); + } + if (container) scope.container = scopeDigest("container", container); + if (!scope.network && !scope.container) return { host: scopeDigest("unverifiable", randomUUID()) }; + return scope; +} + +/** + * Whether a recorded owner scope proves the record was written where this process runs. Only an + * exact, well-formed match counts; anything else (absent, malformed, partial) is "no". + */ +export function sameRuntimeOwnerScope(recorded: unknown, own: RuntimeOwnerScope): boolean { + if (!recorded || typeof recorded !== "object" || Array.isArray(recorded)) return false; + const record = recorded as Record; + if (typeof record.host !== "string" || record.host !== own.host) return false; + if (own.network === undefined && own.container === undefined) { + return record.boot === undefined && record.network === undefined && record.container === undefined; + } + return (own.network !== undefined && record.network === own.network && own.boot !== undefined && record.boot === own.boot) + || (own.container !== undefined && record.container === own.container); +} + export function writePid(pid: number): void { ensureProcessStateDir(); + publishedOwnershipRecords = true; atomicWriteFile(getPidPath(), String(pid)); } @@ -45,9 +161,16 @@ export type RuntimePortState = { /** * The live proxy's port when this runtime is a sibling instance started beside it * (`src/codex/sibling-start.ts`). `ocx stop` reads it to leave shared client routing alone. - * Absent for every other runtime, so those records keep their bytes. + * Absent for every other runtime. */ siblingOfPort?: number; + /** + * {@link runtimeOwnerScope} of the owner, stamped by {@link writeRuntimePort} when this process + * writes its own record. Not validated here: a malformed value must not hide a live owner from + * status, stop or liveness, and the one reader that uses it (`observeProxyLiveness`) accepts + * only an exact match ({@link sameRuntimeOwnerScope}). + */ + ownerScope?: RuntimeOwnerScope; }; export function isValidRuntimePortState(value: unknown): value is RuntimePortState { @@ -70,7 +193,12 @@ export function isValidRuntimePortState(value: unknown): value is RuntimePortSta export function writeRuntimePort(state: RuntimePortState): void { ensureProcessStateDir(); - atomicWriteFile(getRuntimePortPath(), JSON.stringify(state, null, 2) + "\n"); + publishedOwnershipRecords = true; + // Only the owner can say where it runs; a record written on another process's behalf keeps + // whatever identity it already carried. + const ownerScope = state.pid === process.pid ? runtimeOwnerScope() : state.ownerScope; + const record = { ...state, ...(ownerScope === undefined ? {} : { ownerScope }) }; + atomicWriteFile(getRuntimePortPath(), JSON.stringify(record, null, 2) + "\n"); // The record proves this home's owner only to a reader that knows where it lives. // One pointer in the shared registry makes the record findable from every home; // it is best-effort because ownership never depends on the registry write landing. diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 72947685992..f47d3973122 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -12,7 +12,10 @@ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; -import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state"; +import { + hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, + runtimeOwnerScope, sameRuntimeOwnerScope, verifyPidIdentity, type RuntimeOwnerScope, +} from "../config/process-state"; import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { @@ -93,7 +96,9 @@ export interface LivenessIo { * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. */ bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; - readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null; + readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; ownerScope?: unknown } | null; + /** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */ + ownProcessFn?: () => OwnProcess; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; /** @@ -513,21 +518,43 @@ export type ProxyLivenessObservation = | { kind: "live"; live: LiveProxy } | { kind: "absent" | "unknown"; live: null }; +export interface OwnProcess { + pid: number; + /** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */ + publishedOwnership: boolean; + scope: RuntimeOwnerScope; +} + +function currentOwnProcess(): OwnProcess { + return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), scope: runtimeOwnerScope() }; +} + /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { - let alivePid: number | null = null, runtimePid: number | undefined, unknown = false; + let alivePid: number | null = null, runtimePid: number | undefined, runtimeScope: unknown, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, - runtimePid: pid => { runtimePid = pid; }, + runtimePid: (pid, ownerScope) => { runtimePid = pid; runtimeScope = ownerScope; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; + // #8552: in a container the proxy is PID 1 on every start, so the record the previous container + // left names this very process; signalling it succeeds and its command line is ocx. Such a + // record is stale only when this process never published ownership itself (so the writer was an + // earlier lifetime of the PID number) AND the runtime record's owner ran where this process runs + // (same host and boot, plus the same network namespace or container). That scope check keeps a + // RollingUpdate safe: the old pod is also PID 1 and its listener sits in another network + // namespace, which the endpoint probe cannot see. A record without a scope (an older version, + // which may still be serving during an upgrade) keeps the refusal. + const own = (io.ownProcessFn ?? currentOwnProcess)(); + const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && sameRuntimeOwnerScope(runtimeScope, own.scope); const candidates = new Set(); if (alivePid !== null) candidates.add(alivePid); if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); for (const pid of candidates) { if (unknown) break; if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; } + if (ownEarlierLifetime && pid === own.pid) continue; if (pid !== alivePid) { try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; @@ -557,7 +584,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise void; - runtimePid: (pid: number | undefined) => void; + runtimePid: (pid: number | undefined, ownerScope: unknown) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too @@ -659,7 +686,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); - observer?.runtimePid(record?.pid); + observer?.runtimePid(record?.pid, record?.ownerScope); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; diff --git a/structure/runtime.md b/structure/runtime.md index 0a047af9a84..1f938869bcb 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the PID of the process running the recovery recheck, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name plus either the same network namespace within the same boot (a container restarted inside its Kubernetes pod; namespace inode numbers repeat across boots) or the same randomly named writable layer of the overlay mounted at `/` (an `upperdir` in mountinfo with a 64-hex layer id, as Docker's `overlay2` driver and containers/storage use: the same container started again, also across a host reboot; no other container mounts it. Sequence-numbered containerd snapshots and overlay host roots repeat across machines or sandboxes, and a container id is visible to every container joined to that container's network, so none of them is used); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, or another node matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. Processes sharing one network namespace (host networking, pod sidecars, WSL distros in one VM) can match, and there the loopback probe still finds a listening owner; an owner that is alive but no longer listening, an owner exit cleanup that removes records by PID only, sandboxes that fake namespace numbers or clone a whole VM (gVisor, restored Firecracker/Kata snapshots), and non-Linux machines with the same host name syncing one home are residuals for a shared state volume. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 8b3b5f6cc4d..ee6b43cd16f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1225,6 +1225,7 @@ "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-self-pid-record.test.ts": "server", "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server", "proxy-liveness.test.ts": "server", "readyz-features.test.ts": "server", "served-by-header.test.ts": "server", "qoder-adapter.test.ts": "providers", diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts new file mode 100644 index 00000000000..cc567b48446 --- /dev/null +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -0,0 +1,264 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { rootOverlayUpperdirFromMountinfo, runtimeOwnerScope, sameRuntimeOwnerScope } from "../../src/config/process-state"; +import { observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; +import { watchdogMs } from "../helpers/ci-watchdog"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { repoPath } from "../helpers/repo-root"; + +// #8552: a container runs the proxy as PID 1 on every start, so the record the previous container +// left names PID 1 too -- the starting process itself. Signalling it succeeds and its command line +// is ocx, which used to read as a live owner and refused startup on every restart. + +const OCX_COMMAND_LINE = "bun /app/src/cli/index.ts start"; +const refused = (async () => { throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); }) as unknown as typeof fetch; + +function ioFor(extra: Partial): LivenessIo { + return { + readPidFn: () => null, + readRuntimeFn: () => null, + verifyPidFn: pid => pid, + configFn: () => ({ port: 10100, hostname: "127.0.0.1" }), + loopbackAvailabilityFn: () => "available", + readCommandLineFn: () => OCX_COMMAND_LINE, + attempts: 1, + timeoutMs: 50, + fetchFn: refused, + ...extra, + }; +} + +const OWN = { host: "0123456789abcdef0123456789abcdef", boot: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", network: "11111111111111111111111111111111" }; +const unpublished = () => ({ pid: process.pid, publishedOwnership: false, scope: OWN }); +const selfRecord = (ownerScope: unknown = OWN) => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", ...(ownerScope === null ? {} : { ownerScope }) }); + +describe("a record naming this process's own PID", () => { + test("a runtime record an earlier run of this container left is absent", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + + // Docker gives a restarted container a new network namespace, and a host reboot a new boot id, + // but the container keeps its own writable layer. + test("a restarted Docker container matches on its writable layer, even across a host reboot", async () => { + const own = { host: OWN.host, boot: "cccccccccccccccccccccccccccccccc", network: "22222222222222222222222222222222", container: "33333333333333333333333333333333" }; + const io = ioFor({ + readRuntimeFn: () => selfRecord({ host: OWN.host, boot: OWN.boot, network: OWN.network, container: own.container }), + ownProcessFn: () => ({ pid: process.pid, publishedOwnership: false, scope: own }), + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + + test("a pid file beside that runtime record is absent too", async () => { + const io = ioFor({ readPidFn: () => process.pid, readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + + test("a pid file alone carries no owner scope and still refuses", async () => { + const io = ioFor({ readPidFn: () => process.pid, ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + // Every container is PID 1. During a RollingUpdate, or with a fixed hostname and start-first, the + // old container still serves this volume from another network namespace the probe cannot see. + test("a same-host record from another network namespace and container still refuses", async () => { + const io = ioFor({ + readRuntimeFn: () => selfRecord({ host: OWN.host, network: "44444444444444444444444444444444", container: "55555555555555555555555555555555" }), + ownProcessFn: unpublished, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("a record from another host still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord({ ...OWN, host: "fedcba9876543210fedcba9876543210" }), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + // Namespace inode numbers repeat across boots and machines. + test("an equal network namespace from another boot still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord({ ...OWN, boot: "dddddddddddddddddddddddddddddddd" }), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + for (const [label, scope] of [["no owner scope (an older version, maybe still serving)", null], ["a malformed owner scope", OWN.host], ["a host-only scope on Linux", { host: OWN.host }]] as const) { + test(`a record with ${label} still refuses`, async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord(scope), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + } + + test("once this process has published ownership, its own PID still reads as a live owner", async () => { + const io = ioFor({ + readRuntimeFn: () => selfRecord(), + ownProcessFn: () => ({ ...unpublished(), publishedOwnership: true }), + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("another alive ocx PID beside the self record still refuses", async () => { + const io = ioFor({ + readPidFn: () => process.ppid, + readRuntimeFn: pid => pid === undefined ? selfRecord() : null, + ownProcessFn: unpublished, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("an endpoint that did not answer definitively still refuses", async () => { + const io = ioFor({ + readRuntimeFn: () => selfRecord(), + ownProcessFn: unpublished, + fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as unknown as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); +}); + +describe("owner scope comparison", () => { + test("Linux needs the host plus the network namespace within one boot, or the container", () => { + const own = { host: "h", boot: "b", network: "n", container: "c" }; + expect(sameRuntimeOwnerScope({ host: "h", boot: "b", network: "n" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", boot: "x", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", boot: "x", network: "x", container: "c" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", boot: "b", network: "x", container: "y" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "other", boot: "b", network: "n", container: "c" }, own)).toBe(false); + for (const malformed of [undefined, null, "h", ["h"], { host: 1, network: "n" }]) expect(sameRuntimeOwnerScope(malformed, own)).toBe(false); + }); + + test("a machine-wide PID space compares the host alone, and never against a Linux record", () => { + const own = { host: "h" }; + expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", boot: "b" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "other" }, own)).toBe(false); + }); + + test("the container key is the root overlay's own randomly named writable layer, never a shared id", () => { + const docker = "/var/lib/docker/overlay2/" + "a".repeat(64) + "/diff"; + const root = (fstype: string, options: string) => `1001 1000 0:52 / / rw,relatime master:1 - ${fstype} ${fstype} ${options}`; + // A container joined to another's network bind-mounts that one's hostname; it must not count. + const joined = `1201 1001 8:1 /var/lib/docker/containers/${"b".repeat(64)}/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw`; + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `ro,lowerdir=/l1:/l2,upperdir=${docker},workdir=/w`), joined].join("\n") + "\n")).toBe(docker); + expect(rootOverlayUpperdirFromMountinfo(joined + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("ext4", "rw,errors=remount-ro") + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("fuse.fuse-overlayfs", "rw,user_id=0") + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l1") + "\n")).toBeUndefined(); + const podman = "/var/lib/containers/storage/overlay/" + "c".repeat(64) + "/diff"; + expect(rootOverlayUpperdirFromMountinfo(root("overlay", `rw,lowerdir=/l,upperdir=${podman},workdir=/w`))).toBe(podman); + // Sequence-numbered containerd snapshots repeat on nodes built alike; an overlay host root is + // shared by every sandbox using it. Neither proves anything beyond one machine. + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,workdir=/w"))).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l,upperdir=/media/root-rw/overlay,workdir=/w"))).toBeUndefined(); + // The visible root is the last one mounted. + const older = "/var/lib/docker/overlay2/" + "d".repeat(64) + "/diff"; + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `rw,upperdir=${older},workdir=/w`), root("overlay", `rw,upperdir=${docker},workdir=/w`)].join("\n"))).toBe(docker); + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `rw,upperdir=${older},workdir=/w`), root("overlay", "rw,upperdir=/var/lib/containerd/snapshots/7/fs,workdir=/w")].join("\n"))).toBeUndefined(); + }); +}); + +const roots: string[] = []; +const children: Array> = []; +afterEach(async () => { + while (children.length) { + const child = children.pop()!; + if (child.exitCode === null) child.kill(); + await child.exited; + } + while (roots.length) removeTreeWithRetry(roots.pop()!); +}); + +async function freePort(): Promise { + const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") }); + const port = probe.port!; + await probe.stop(true); + return port; +} + +const START_WAIT_MS = watchdogMs(15_000); + +// `exec` keeps the shell's PID, so the shell plays the previous container and `ocx start` the next +// one: same PID, new lifetime, records naming it already on disk. Windows has no exec handoff. +async function startAfterOwnLifetime(ownerScope: unknown): Promise<{ child: ReturnType; port: number; ocxHome: string; served: boolean; stderr: Promise }> { + const root = mkdtempSync(join(tmpdir(), "ocx-self-pid-")); + roots.push(root); + const ocxHome = join(root, "ocx"), codexHome = join(root, "codex"), home = join(root, "home"), runtime = join(root, "runtime"); + for (const path of [ocxHome, codexHome, home, runtime]) mkdirSync(path, { recursive: true }); + const port = await freePort(); + writeFileSync(join(ocxHome, "config.json"), JSON.stringify({ + port, + hostname: "127.0.0.1", + codexAutoStart: false, + syncResumeHistory: false, + clientIntegrations: { codex: false, grok: false, "claude-desktop": false }, + claudeCode: { systemEnv: false }, + providers: {}, + defaultProvider: "openai", + })); + const script = [ + 'printf %s "$$" > "$OPENCODEX_HOME/ocx.pid"', + 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1","ownerScope":%s}\' "$$" "$SELF_PID_PORT" "$SELF_PID_SCOPE" > "$OPENCODEX_HOME/runtime-port.json"', + 'exec "$SELF_PID_BUN" "$SELF_PID_CLI" start --port "$SELF_PID_PORT"', + ].join(" && "); + const child = Bun.spawn(["/bin/sh", "-c", script], { + cwd: root, + env: { + HOME: home, + USERPROFILE: home, + CODEX_HOME: codexHome, + OPENCODEX_HOME: ocxHome, + XDG_RUNTIME_DIR: runtime, + NO_PROXY: "127.0.0.1,localhost", + PATH: process.env.PATH ?? "/usr/bin:/bin", + SELF_PID_BUN: process.execPath, + SELF_PID_CLI: repoPath("src/cli/index.ts"), + SELF_PID_PORT: String(port), + SELF_PID_SCOPE: JSON.stringify(ownerScope), + }, + stdout: "pipe", + stderr: "pipe", + }); + children.push(child); + const stderr = new Response(child.stderr).text(); + const deadline = Date.now() + START_WAIT_MS; + let served = false; + while (Date.now() < deadline && child.exitCode === null && !served) { + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`, { signal: AbortSignal.timeout(500) }); + const body = await response.json() as { pid?: number }; + served = response.ok && body.pid === child.pid; + } catch { /* not listening yet */ } + if (!served) await Bun.sleep(50); + } + return { child, port, ocxHome, served, stderr }; +} + +const posix = process.platform !== "win32"; + +test.skipIf(!posix)("ocx start recovers records an earlier run of its PID left in the same place", async () => { + const { child, port, ocxHome, served, stderr } = await startAfterOwnLifetime(runtimeOwnerScope()); + if (!served) { + if (child.exitCode === null) child.kill(); + await child.exited; + const text = await stderr; + const reason = text.split("\n").find(line => line.startsWith("error:")) ?? text.trim().slice(0, 400); + throw new Error(`start did not serve as PID ${child.pid}: exit ${child.exitCode}: ${reason}`); + } + const record = JSON.parse(readFileSync(join(ocxHome, "runtime-port.json"), "utf8")) as { pid: number; port: number; attestationSecret?: string; ownerScope?: unknown }; + expect(record.pid).toBe(child.pid); + expect(record.port).toBe(port); + // A fresh publication: the stale record had no attestation secret. + expect(typeof record.attestationSecret).toBe("string"); + expect(record.ownerScope).toEqual(runtimeOwnerScope()); +}, Math.max(30_000, START_WAIT_MS * 2)); + +test.skipIf(!posix)("ocx start still refuses a same-PID record written somewhere else", async () => { + const { child, served, stderr, ocxHome } = await startAfterOwnLifetime({ ...runtimeOwnerScope(), host: "fedcba9876543210fedcba9876543210" }); + expect(served).toBe(false); + expect(await child.exited).toBe(1); + expect(await stderr).toContain("Proxy ownership is unverified"); + expect(existsSync(join(ocxHome, "ocx.pid"))).toBe(true); +}, Math.max(30_000, START_WAIT_MS * 2)); diff --git a/tests/service/process-state.test.ts b/tests/service/process-state.test.ts index 02cf8a3d6f8..d2669745cd7 100644 --- a/tests/service/process-state.test.ts +++ b/tests/service/process-state.test.ts @@ -15,6 +15,7 @@ import { readRuntimePort, removePid, removeRuntimePort, + runtimeOwnerScope, setOcxStartProcessCacheForTests, setProcessCommandLineExecForTests, setProcessCommandLineNativeForTests, @@ -393,6 +394,20 @@ describe("proxy process-state ownership", () => { expect(readRuntimePort(9999)).toBeNull(); }); + // #8552: the scope says where the OWNER runs, so only the owner may stamp it. + test("a runtime record this process writes for itself carries its owner scope; one for another PID keeps its own", () => { + const scope = runtimeOwnerScope(); + expect(scope.host).toMatch(/^[0-9a-f]{32}$/); + expect(runtimeOwnerScope()).toBe(scope); + if (process.platform !== "linux") expect(scope).toEqual({ host: scope.host }); + writeRuntimePort({ pid: process.pid, port: 58195 }); + expect(readRuntimePort()?.ownerScope).toEqual(scope); + writeRuntimePort({ pid: 1234, port: 58195, ownerScope: { host: "kept-as-written" } }); + expect(readRuntimePort()?.ownerScope).toEqual({ host: "kept-as-written" }); + writeRuntimePort({ pid: 1234, port: 58195 }); + expect(readRuntimePort()?.ownerScope).toBeUndefined(); + }); + test("runtime port removal preserves newer pid state", () => { writeRuntimePort({ pid: 1234, port: 58195 }); removeRuntimePort(9999);