From f2b3d3018876debf0398d93ed463888e56b2a83d Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 12 Nov 2026 00:38:48 +0900 Subject: [PATCH 1/5] test(start): reproduce #8552 self-PID record refusal --- .../proxy-liveness-self-pid-record.test.ts | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 tests/server/proxy-liveness-self-pid-record.test.ts diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts new file mode 100644 index 00000000000..2b0ed4553c0 --- /dev/null +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; +import { watchdogMs } from "../helpers/ci-watchdog"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { repoPath } from "../helpers/repo-root"; + +// #8552: a container runs the proxy as PID 1 on every start, so the record the previous container +// left names PID 1 too -- the starting process itself. Signalling it succeeds and its command line +// is ocx, which used to read as a live owner and refused startup on every restart. + +const OCX_COMMAND_LINE = "bun /app/src/cli/index.ts start"; +const refused = (async () => { throw Object.assign(new Error("refused"), { code: "ECONNREFUSED" }); }) as unknown as typeof fetch; + +function ioFor(extra: Partial): LivenessIo { + return { + readPidFn: () => null, + readRuntimeFn: () => null, + verifyPidFn: pid => pid, + configFn: () => ({ port: 10100, hostname: "127.0.0.1" }), + loopbackAvailabilityFn: () => "available", + readCommandLineFn: () => OCX_COMMAND_LINE, + attempts: 1, + timeoutMs: 50, + fetchFn: refused, + ...extra, + }; +} + +const unpublished = () => ({ pid: process.pid, publishedOwnership: false }); + +describe("a record naming this process's own PID", () => { + test("a runtime record left by an earlier lifetime of this PID is absent", async () => { + const io = ioFor({ readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + + test("a pid file left by an earlier lifetime of this PID is absent", async () => { + const io = ioFor({ readPidFn: () => process.pid, ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + + test("once this process has published ownership, its own PID still reads as a live owner", async () => { + const io = ioFor({ + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), + ownProcessFn: () => ({ pid: process.pid, publishedOwnership: true }), + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("another alive ocx PID beside the self record still refuses", async () => { + const io = ioFor({ + readPidFn: () => process.ppid, + readRuntimeFn: pid => pid === undefined ? { pid: process.pid, port: 10100, hostname: "127.0.0.1" } : null, + ownProcessFn: unpublished, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("an endpoint that did not answer definitively still refuses", async () => { + const io = ioFor({ + readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), + ownProcessFn: unpublished, + fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as unknown as typeof fetch, + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); +}); + +const roots: string[] = []; +const children: Array> = []; +afterEach(async () => { + while (children.length) { + const child = children.pop()!; + if (child.exitCode === null) child.kill(); + await child.exited; + } + while (roots.length) removeTreeWithRetry(roots.pop()!); +}); + +async function freePort(): Promise { + const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("") }); + const port = probe.port!; + await probe.stop(true); + return port; +} + +const START_WAIT_MS = watchdogMs(15_000); + +// `exec` keeps the shell's PID, so the shell plays the previous container and `ocx start` the next +// one: same PID, new lifetime, records naming it already on disk. Windows has no exec handoff. +test.skipIf(process.platform === "win32")("ocx start recovers records its own PID left in an earlier lifetime", async () => { + const root = mkdtempSync(join(tmpdir(), "ocx-self-pid-")); + roots.push(root); + const ocxHome = join(root, "ocx"), codexHome = join(root, "codex"), home = join(root, "home"), runtime = join(root, "runtime"); + for (const path of [ocxHome, codexHome, home, runtime]) mkdirSync(path, { recursive: true }); + const port = await freePort(); + writeFileSync(join(ocxHome, "config.json"), JSON.stringify({ + port, + hostname: "127.0.0.1", + codexAutoStart: false, + syncResumeHistory: false, + clientIntegrations: { codex: false, grok: false, "claude-desktop": false }, + claudeCode: { systemEnv: false }, + providers: {}, + defaultProvider: "openai", + })); + const script = [ + 'printf %s "$$" > "$OPENCODEX_HOME/ocx.pid"', + 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1"}\' "$$" "$SELF_PID_PORT" > "$OPENCODEX_HOME/runtime-port.json"', + 'exec "$SELF_PID_BUN" "$SELF_PID_CLI" start --port "$SELF_PID_PORT"', + ].join(" && "); + const child = Bun.spawn(["/bin/sh", "-c", script], { + cwd: root, + env: { + HOME: home, + USERPROFILE: home, + CODEX_HOME: codexHome, + OPENCODEX_HOME: ocxHome, + XDG_RUNTIME_DIR: runtime, + NO_PROXY: "127.0.0.1,localhost", + PATH: process.env.PATH ?? "/usr/bin:/bin", + SELF_PID_BUN: process.execPath, + SELF_PID_CLI: repoPath("src/cli/index.ts"), + SELF_PID_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); + children.push(child); + const stderr = new Response(child.stderr).text(); + const deadline = Date.now() + START_WAIT_MS; + let served: { pid?: number } | null = null; + while (Date.now() < deadline && child.exitCode === null && !served) { + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`, { signal: AbortSignal.timeout(500) }); + const body = await response.json() as { pid?: number }; + if (response.ok && body.pid === child.pid) served = body; + } catch { /* not listening yet */ } + if (!served) await Bun.sleep(50); + } + if (!served) { + if (child.exitCode === null) child.kill(); + await child.exited; + const reason = (await stderr).split("\n").find(line => line.startsWith("error:")) ?? (await stderr).trim().slice(0, 400); + throw new Error(`start did not serve as PID ${child.pid}: exit ${child.exitCode}: ${reason}`); + } + expect(served.pid).toBe(child.pid); + const runtimePath = join(ocxHome, "runtime-port.json"); + expect(existsSync(runtimePath)).toBe(true); + const record = JSON.parse(readFileSync(runtimePath, "utf8")) as { pid: number; port: number; attestationSecret?: string }; + expect(record.pid).toBe(child.pid); + expect(record.port).toBe(port); + // A fresh publication: the stale record had no attestation secret. + expect(typeof record.attestationSecret).toBe("string"); +}, Math.max(30_000, START_WAIT_MS * 2)); From 46207d0a9a6e5bed85b587db838507b6cc2a2eca Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 12 Nov 2026 00:59:16 +0900 Subject: [PATCH 2/5] fix(start): recover a same-host record left by an earlier lifetime of this PID (#8552) --- .../test-layout/layout-start-ownership.json | 1 + src/config/process-state.ts | 38 +++++++++- src/server/proxy-liveness.ts | 38 ++++++++-- structure/runtime.md | 2 +- tests/fixtures/test-layout-expected.json | 1 + tests/server/config.test.ts | 5 +- .../proxy-liveness-self-pid-record.test.ts | 71 ++++++++++++++----- tests/service/process-state.test.ts | 15 ++-- 8 files changed, 135 insertions(+), 36 deletions(-) diff --git a/scripts/test-layout/layout-start-ownership.json b/scripts/test-layout/layout-start-ownership.json index 7caaff72af5..c8e16c53002 100644 --- a/scripts/test-layout/layout-start-ownership.json +++ b/scripts/test-layout/layout-start-ownership.json @@ -4,6 +4,7 @@ "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-self-pid-record.test.ts": "server", "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server" } diff --git a/src/config/process-state.ts b/src/config/process-state.ts index b8732cc50fc..85afe39b516 100644 --- a/src/config/process-state.ts +++ b/src/config/process-state.ts @@ -1,5 +1,7 @@ import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; +import { hostname } from "node:os"; import { join } from "node:path"; import { isLocalAttestationSecret } from "../lib/local-management-attestation"; import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; @@ -27,8 +29,33 @@ function ensureProcessStateDir(): void { else hardenConfigDir(); } +/** + * Whether this process has written `ocx.pid` or `runtime-port.json` in its own lifetime (#8552). + * + * Set before the write and never cleared: a record this process wrote and later removed still + * means its PID once named a live owner of this home, so it can never be told apart from an + * earlier lifetime of the same PID number. + */ +let publishedOwnershipRecords = false; + +export function hasPublishedOwnershipRecords(): boolean { + return publishedOwnershipRecords; +} + +/** + * Which host name a runtime record was written under, without storing the name itself (#8552). + * + * Every container runs the proxy as PID 1, so a PID alone cannot say whether a record came from + * this container's previous lifetime or from another pod serving the same state volume. A + * restarted container keeps its pod's host name; a replacement pod does not. + */ +export function runtimeHostIdentity(): string { + return createHash("sha256").update(hostname()).digest("hex").slice(0, 32); +} + export function writePid(pid: number): void { ensureProcessStateDir(); + publishedOwnershipRecords = true; atomicWriteFile(getPidPath(), String(pid)); } @@ -45,9 +72,15 @@ export type RuntimePortState = { /** * The live proxy's port when this runtime is a sibling instance started beside it * (`src/codex/sibling-start.ts`). `ocx stop` reads it to leave shared client routing alone. - * Absent for every other runtime, so those records keep their bytes. + * Absent for every other runtime. */ siblingOfPort?: number; + /** + * {@link runtimeHostIdentity} of the writer, stamped by {@link writeRuntimePort}. Not validated + * here: a malformed value must not hide a live owner from status, stop or liveness, and the one + * reader that uses it (`observeProxyLiveness`) accepts only an exact match. + */ + hostIdentity?: string; }; export function isValidRuntimePortState(value: unknown): value is RuntimePortState { @@ -70,7 +103,8 @@ export function isValidRuntimePortState(value: unknown): value is RuntimePortSta export function writeRuntimePort(state: RuntimePortState): void { ensureProcessStateDir(); - atomicWriteFile(getRuntimePortPath(), JSON.stringify(state, null, 2) + "\n"); + publishedOwnershipRecords = true; + atomicWriteFile(getRuntimePortPath(), JSON.stringify({ ...state, hostIdentity: runtimeHostIdentity() }, null, 2) + "\n"); // The record proves this home's owner only to a reader that knows where it lives. // One pointer in the shared registry makes the record findable from every home; // it is best-effort because ownership never depends on the registry write landing. diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 72947685992..0da0ca1d7dd 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -12,7 +12,10 @@ import { loadConfig } from "../config"; import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target"; import { normalizeBindHostname } from "./bind-host"; -import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state"; +import { + hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, + runtimeHostIdentity, verifyPidIdentity, +} from "../config/process-state"; import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; import { @@ -93,7 +96,9 @@ export interface LivenessIo { * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. */ bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; - readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null; + readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; hostIdentity?: string } | null; + /** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */ + ownProcessFn?: () => OwnProcess; configFn?: () => { port?: number; hostname?: string }; timeoutMs?: number; /** @@ -513,21 +518,42 @@ export type ProxyLivenessObservation = | { kind: "live"; live: LiveProxy } | { kind: "absent" | "unknown"; live: null }; +export interface OwnProcess { + pid: number; + /** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */ + publishedOwnership: boolean; + hostIdentity: string; +} + +function currentOwnProcess(): OwnProcess { + return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), hostIdentity: runtimeHostIdentity() }; +} + /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { - let alivePid: number | null = null, runtimePid: number | undefined, unknown = false; + let alivePid: number | null = null, runtimePid: number | undefined, runtimeHost: unknown, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, - runtimePid: pid => { runtimePid = pid; }, + runtimePid: (pid, hostIdentity) => { runtimePid = pid; runtimeHost = hostIdentity; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; + // #8552: in a container the proxy is PID 1 on every start, so the record the previous container + // left names this very process; signalling it succeeds and its command line is ocx. Such a + // record is stale only when this process never published ownership itself (so the writer was an + // earlier lifetime of the PID number) AND the runtime record was written under this host name. + // The host check is what keeps a RollingUpdate safe: the old pod is also PID 1 and its listener + // sits in another network namespace, but its pod name differs. A record without the identity + // (an older version, which may still be serving during an upgrade) keeps the refusal. + const own = (io.ownProcessFn ?? currentOwnProcess)(); + const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && runtimeHost === own.hostIdentity; const candidates = new Set(); if (alivePid !== null) candidates.add(alivePid); if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); for (const pid of candidates) { if (unknown) break; if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; } + if (ownEarlierLifetime && pid === own.pid) continue; if (pid !== alivePid) { try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; @@ -557,7 +583,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise void; - runtimePid: (pid: number | undefined) => void; + runtimePid: (pid: number | undefined, hostIdentity: unknown) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too @@ -659,7 +685,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); - observer?.runtimePid(record?.pid); + observer?.runtimePid(record?.pid, record?.hostIdentity); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; diff --git a/structure/runtime.md b/structure/runtime.md index 0a047af9a84..571beb40a77 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the starting process's own PID, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with a `hostIdentity` equal to this host's (`runtimeHostIdentity`, a SHA-256 prefix of the host name that `writeRuntimePort` stamps). A restarted container keeps its pod's host name; a replacement pod, which can start while the old pod still serves the same volume from another network namespace, does not, so its refusal stands. A record without the identity (an older version), a PID file without a runtime record, and a malformed identity keep the refusal; the identity is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 8b3b5f6cc4d..ee6b43cd16f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1225,6 +1225,7 @@ "proxy-liveness-recovery.test.ts": "server", "proxy-liveness-owner-evidence.test.ts": "server", "proxy-liveness-bind-scope.test.ts": "server", + "proxy-liveness-self-pid-record.test.ts": "server", "proxy-liveness-nonloopback-bind-proof.test.ts": "server", "system-restart-ownership.test.ts": "server", "proxy-liveness.test.ts": "server", "readyz-features.test.ts": "server", "served-by-header.test.ts": "server", "qoder-adapter.test.ts": "providers", diff --git a/tests/server/config.test.ts b/tests/server/config.test.ts index f9a4c4915fa..e30fce6dc04 100644 --- a/tests/server/config.test.ts +++ b/tests/server/config.test.ts @@ -35,6 +35,7 @@ import { import * as windowsAcl from "../../src/lib/windows-secret-acl"; import { setTrustedWindowsSystemDirectoryResolverForTests } from "../../src/lib/windows-elevation"; import { AtomicWriteResidualTempError, atomicWriteFile, atomicWriteFileAsync, hardenConfigDir, hardenExistingSecret, renameAtomicFile, saveConfig } from "../../src/config"; +import { runtimeHostIdentity } from "../../src/config/process-state"; import { nextAtomicTempSequence } from "../../src/config/atomic-write"; import { flushConfigDirHardeningForTests } from "../../src/config/paths"; import { @@ -3122,8 +3123,8 @@ describe("opencodex config defaults", () => { const attestationSecret = "A".repeat(43); writeRuntimePort({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); + expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); expect(readRuntimePort(9999)).toBeNull(); }); diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts index 2b0ed4553c0..c2bf83a96f7 100644 --- a/tests/server/proxy-liveness-self-pid-record.test.ts +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { runtimeHostIdentity } from "../../src/config/process-state"; import { observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; import { watchdogMs } from "../helpers/ci-watchdog"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -29,23 +30,42 @@ function ioFor(extra: Partial): LivenessIo { }; } -const unpublished = () => ({ pid: process.pid, publishedOwnership: false }); +const HOST = "0123456789abcdef0123456789abcdef"; +const unpublished = () => ({ pid: process.pid, publishedOwnership: false, hostIdentity: HOST }); +const selfRecord = (hostIdentity: string | null = HOST) => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", ...(hostIdentity === null ? {} : { hostIdentity }) }); describe("a record naming this process's own PID", () => { - test("a runtime record left by an earlier lifetime of this PID is absent", async () => { - const io = ioFor({ readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), ownProcessFn: unpublished }); + test("a runtime record this host's earlier lifetime of this PID left is absent", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); }); - test("a pid file left by an earlier lifetime of this PID is absent", async () => { - const io = ioFor({ readPidFn: () => process.pid, ownProcessFn: unpublished }); + test("a pid file beside that runtime record is absent too", async () => { + const io = ioFor({ readPidFn: () => process.pid, readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); }); + test("a pid file alone carries no host identity and still refuses", async () => { + const io = ioFor({ readPidFn: () => process.pid, ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + // Every container is PID 1: during a RollingUpdate the old pod still serves this state volume + // from another network namespace, and only its host name tells it apart. + test("a record another host wrote under the same PID number still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord("fedcba9876543210fedcba9876543210"), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + + test("a record without a host identity (an older version, maybe still serving) still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord(null), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + test("once this process has published ownership, its own PID still reads as a live owner", async () => { const io = ioFor({ - readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), - ownProcessFn: () => ({ pid: process.pid, publishedOwnership: true }), + readRuntimeFn: () => selfRecord(), + ownProcessFn: () => ({ ...unpublished(), publishedOwnership: true }), }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); @@ -53,7 +73,7 @@ describe("a record naming this process's own PID", () => { test("another alive ocx PID beside the self record still refuses", async () => { const io = ioFor({ readPidFn: () => process.ppid, - readRuntimeFn: pid => pid === undefined ? { pid: process.pid, port: 10100, hostname: "127.0.0.1" } : null, + readRuntimeFn: pid => pid === undefined ? selfRecord() : null, ownProcessFn: unpublished, }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); @@ -61,7 +81,7 @@ describe("a record naming this process's own PID", () => { test("an endpoint that did not answer definitively still refuses", async () => { const io = ioFor({ - readRuntimeFn: () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1" }), + readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished, fetchFn: (async () => { throw Object.assign(new Error("timed out"), { name: "TimeoutError" }); }) as unknown as typeof fetch, }); @@ -91,7 +111,7 @@ const START_WAIT_MS = watchdogMs(15_000); // `exec` keeps the shell's PID, so the shell plays the previous container and `ocx start` the next // one: same PID, new lifetime, records naming it already on disk. Windows has no exec handoff. -test.skipIf(process.platform === "win32")("ocx start recovers records its own PID left in an earlier lifetime", async () => { +async function startAfterOwnLifetime(hostIdentity: string): Promise<{ child: ReturnType; port: number; ocxHome: string; served: boolean; stderr: Promise }> { const root = mkdtempSync(join(tmpdir(), "ocx-self-pid-")); roots.push(root); const ocxHome = join(root, "ocx"), codexHome = join(root, "codex"), home = join(root, "home"), runtime = join(root, "runtime"); @@ -109,7 +129,7 @@ test.skipIf(process.platform === "win32")("ocx start recovers records its own PI })); const script = [ 'printf %s "$$" > "$OPENCODEX_HOME/ocx.pid"', - 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1"}\' "$$" "$SELF_PID_PORT" > "$OPENCODEX_HOME/runtime-port.json"', + 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1","hostIdentity":"%s"}\' "$$" "$SELF_PID_PORT" "$SELF_PID_HOST" > "$OPENCODEX_HOME/runtime-port.json"', 'exec "$SELF_PID_BUN" "$SELF_PID_CLI" start --port "$SELF_PID_PORT"', ].join(" && "); const child = Bun.spawn(["/bin/sh", "-c", script], { @@ -125,6 +145,7 @@ test.skipIf(process.platform === "win32")("ocx start recovers records its own PI SELF_PID_BUN: process.execPath, SELF_PID_CLI: repoPath("src/cli/index.ts"), SELF_PID_PORT: String(port), + SELF_PID_HOST: hostIdentity, }, stdout: "pipe", stderr: "pipe", @@ -132,27 +153,41 @@ test.skipIf(process.platform === "win32")("ocx start recovers records its own PI children.push(child); const stderr = new Response(child.stderr).text(); const deadline = Date.now() + START_WAIT_MS; - let served: { pid?: number } | null = null; + let served = false; while (Date.now() < deadline && child.exitCode === null && !served) { try { const response = await fetch(`http://127.0.0.1:${port}/healthz`, { signal: AbortSignal.timeout(500) }); const body = await response.json() as { pid?: number }; - if (response.ok && body.pid === child.pid) served = body; + served = response.ok && body.pid === child.pid; } catch { /* not listening yet */ } if (!served) await Bun.sleep(50); } + return { child, port, ocxHome, served, stderr }; +} + +const posix = process.platform !== "win32"; + +test.skipIf(!posix)("ocx start recovers records this host's earlier lifetime of its PID left", async () => { + const { child, port, ocxHome, served, stderr } = await startAfterOwnLifetime(runtimeHostIdentity()); if (!served) { if (child.exitCode === null) child.kill(); await child.exited; - const reason = (await stderr).split("\n").find(line => line.startsWith("error:")) ?? (await stderr).trim().slice(0, 400); + const text = await stderr; + const reason = text.split("\n").find(line => line.startsWith("error:")) ?? text.trim().slice(0, 400); throw new Error(`start did not serve as PID ${child.pid}: exit ${child.exitCode}: ${reason}`); } - expect(served.pid).toBe(child.pid); - const runtimePath = join(ocxHome, "runtime-port.json"); - expect(existsSync(runtimePath)).toBe(true); - const record = JSON.parse(readFileSync(runtimePath, "utf8")) as { pid: number; port: number; attestationSecret?: string }; + const record = JSON.parse(readFileSync(join(ocxHome, "runtime-port.json"), "utf8")) as { pid: number; port: number; attestationSecret?: string; hostIdentity?: string }; expect(record.pid).toBe(child.pid); expect(record.port).toBe(port); // A fresh publication: the stale record had no attestation secret. expect(typeof record.attestationSecret).toBe("string"); + expect(record.hostIdentity).toBe(runtimeHostIdentity()); +}, Math.max(30_000, START_WAIT_MS * 2)); + +test.skipIf(!posix)("ocx start still refuses a same-PID record another host wrote", async () => { + const { child, served, stderr, ocxHome } = await startAfterOwnLifetime("fedcba9876543210fedcba9876543210"); + expect(served).toBe(false); + expect(await child.exited).toBe(1); + expect(await stderr).toContain("Proxy ownership is unverified"); + expect(existsSync(join(ocxHome, "ocx.pid"))).toBe(true); }, Math.max(30_000, START_WAIT_MS * 2)); diff --git a/tests/service/process-state.test.ts b/tests/service/process-state.test.ts index 02cf8a3d6f8..7b262824aa7 100644 --- a/tests/service/process-state.test.ts +++ b/tests/service/process-state.test.ts @@ -15,6 +15,7 @@ import { readRuntimePort, removePid, removeRuntimePort, + runtimeHostIdentity, setOcxStartProcessCacheForTests, setProcessCommandLineExecForTests, setProcessCommandLineNativeForTests, @@ -388,8 +389,8 @@ describe("proxy process-state ownership", () => { const attestationSecret = "A".repeat(43); writeRuntimePort({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); + expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); expect(readRuntimePort(9999)).toBeNull(); }); @@ -413,12 +414,12 @@ describe("proxy process-state ownership", () => { expect(readRuntimePort()).toBeNull(); }); - test("a sibling record carries the live owner's port; every other record keeps its bytes", () => { - // Absent means "not a sibling", and the writer must not add the key: a non-sibling record is - // byte-identical to the one written before the field existed. + test("a sibling record carries the live owner's port; every other record omits the field", () => { + // Absent means "not a sibling", and the writer must not add the key. Every record carries the + // writer's host identity (#8552). writeRuntimePort({ pid: 1234, port: 58195, hostname: "127.0.0.1", ...siblingRuntimeField() }); expect(readFileSync(getRuntimePortPath(), "utf-8")) - .toBe(`${JSON.stringify({ pid: 1234, port: 58195, hostname: "127.0.0.1" }, null, 2)}\n`); + .toBe(`${JSON.stringify({ pid: 1234, port: 58195, hostname: "127.0.0.1", hostIdentity: runtimeHostIdentity() }, null, 2)}\n`); expect(readRuntimePort()?.siblingOfPort).toBeUndefined(); markSiblingStart(10100); @@ -427,7 +428,7 @@ describe("proxy process-state ownership", () => { } finally { resetSiblingStartForTests(); } - expect(readRuntimePort()).toEqual({ pid: 1234, port: 10199, hostname: "127.0.0.1", siblingOfPort: 10100 }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 10199, hostname: "127.0.0.1", siblingOfPort: 10100, hostIdentity: runtimeHostIdentity() }); for (const siblingOfPort of [0, 70000, 1.5, "10100", null]) { writeFileSync(getRuntimePortPath(), JSON.stringify({ pid: 1234, port: 10199, siblingOfPort }), "utf-8"); From 64e23ec8e72b4e3cc4012bdd14135ec209e6cc18 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 12 Nov 2026 01:23:43 +0900 Subject: [PATCH 3/5] fix(start): scope the self-PID proof to the same network namespace or container --- .../src/content/docs/guides/remote-hub.md | 17 +++ src/config/process-state.ts | 100 +++++++++++++++--- src/server/proxy-liveness.ts | 27 ++--- structure/runtime.md | 2 +- tests/server/config.test.ts | 5 +- .../proxy-liveness-self-pid-record.test.ts | 88 +++++++++++---- tests/service/process-state.test.ts | 30 ++++-- 7 files changed, 208 insertions(+), 61 deletions(-) diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index dc6c9e6e7fd..ed2d2b648ed 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -745,6 +745,23 @@ both named volumes and a synthetic catalog survive replacement. This check does real provider account, OAuth callback, custom mount migration, or every CPU architecture; perform the authenticated routed-response check above for your deployment. +### Restarts after an unclean exit + +A container runs the proxy as PID 1 on every start. After an unclean exit (an OOM kill, a crash, a +node failure) the state volume can still hold `ocx.pid` and `runtime-port.json` naming PID 1, and +the next start has to decide whether that owner is gone. It recovers on its own when the record was +written by the same container in an earlier run: a container restarted in place (`docker restart` +of the same container, or a Kubernetes container restart inside its pod) on the same machine, by +an OpenCodex version that records where its owner runs. + +Every other case refuses with `Proxy ownership is unverified; startup refused`, because the old +owner could still be serving the same volume from another container: a replacement Kubernetes pod, +a recreated Compose container, another node, or a record written by an older version (the first +restart after an upgrade). Confirm that nothing else serves this state directory, then move +`runtime-port.json` and `ocx.pid` aside once and start again. Run a single writer per state +volume: a Kubernetes `Deployment` that shares a volume should use `strategy: Recreate` (or +`maxSurge: 0`) so the old pod stops before the new one starts. + ## Rollback Inspect existing Serve mappings before changing them. `tailscale serve reset` removes every mapping diff --git a/src/config/process-state.ts b/src/config/process-state.ts index 85afe39b516..ac6b15257b0 100644 --- a/src/config/process-state.ts +++ b/src/config/process-state.ts @@ -1,6 +1,6 @@ import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, readlinkSync, unlinkSync } from "node:fs"; import { hostname } from "node:os"; import { join } from "node:path"; import { isLocalAttestationSecret } from "../lib/local-management-attestation"; @@ -32,9 +32,10 @@ function ensureProcessStateDir(): void { /** * Whether this process has written `ocx.pid` or `runtime-port.json` in its own lifetime (#8552). * - * Set before the write and never cleared: a record this process wrote and later removed still - * means its PID once named a live owner of this home, so it can never be told apart from an - * earlier lifetime of the same PID number. + * Set before the write, deliberately even when the write then throws, and never cleared: a record + * this process wrote (or may have written) and later removed still means its PID once named a live + * owner of this home, so it can never be told apart from an earlier lifetime of the same PID number. + * Moving the assignment after the write would let a failed write fail open. */ let publishedOwnershipRecords = false; @@ -43,14 +44,78 @@ export function hasPublishedOwnershipRecords(): boolean { } /** - * Which host name a runtime record was written under, without storing the name itself (#8552). + * Where the owner of a runtime record ran, as digests that store none of their inputs (#8552). * * Every container runs the proxy as PID 1, so a PID alone cannot say whether a record came from - * this container's previous lifetime or from another pod serving the same state volume. A - * restarted container keeps its pod's host name; a replacement pod does not. + * this container's previous run or from another container serving the same state volume. On + * Linux, `host` covers the host name and boot id, and at least one of two stronger facts must + * also match ({@link sameRuntimeOwnerScope}): + * + * - `network`, the network namespace: a container restarted inside its Kubernetes pod keeps it, + * because the pod sandbox owns it; + * - `container`, the Docker/Podman container (or CRI-O pod sandbox) id read from mountinfo: a + * restarted container keeps it, and one id never runs twice at the same time. + * + * A replacement pod, a second container with a fixed `hostname`, another node, or a WSL distro + * sharing a Windows home matches neither. Elsewhere the PID space is the whole machine and the host + * name suffices. Linux evidence that cannot be read yields a per-process value that matches no + * other lifetime, so the self-record proof fails closed. */ -export function runtimeHostIdentity(): string { - return createHash("sha256").update(hostname()).digest("hex").slice(0, 32); +export interface RuntimeOwnerScope { + host: string; + network?: string; + container?: string; +} + +let cachedRuntimeOwnerScope: RuntimeOwnerScope | undefined; + +export function runtimeOwnerScope(): RuntimeOwnerScope { + cachedRuntimeOwnerScope ??= computeRuntimeOwnerScope(); + return cachedRuntimeOwnerScope; +} + +function scopeDigest(...parts: string[]): string { + return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32); +} + +const MOUNTINFO_CONTAINER_ID = /\/(?:containers|overlay-containers)\/([0-9a-f]{64})\/(?:userdata\/)?(?:resolv\.conf|hostname|hosts)(?:\s|$)/m; + +/** The container id Docker or Podman bind-mounts `/etc/hostname` and friends from, if any. */ +export function containerIdFromMountinfo(mountinfo: string): string | undefined { + return MOUNTINFO_CONTAINER_ID.exec(mountinfo)?.[1]; +} + +function readOptional(read: () => string): string | undefined { + try { return read(); } catch { return undefined; } +} + +function computeRuntimeOwnerScope(): RuntimeOwnerScope { + if (process.platform !== "linux") return { host: scopeDigest(process.platform, hostname()) }; + const boot = readOptional(() => readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()); + const network = readOptional(() => readlinkSync("/proc/self/ns/net")); + const container = readOptional(() => containerIdFromMountinfo(readFileSync("/proc/self/mountinfo", "utf8")) ?? ""); + const scope: RuntimeOwnerScope = { host: "" }; + if (network && /^net:\[\d+\]$/.test(network)) scope.network = scopeDigest("net", network); + if (container) scope.container = scopeDigest("container", container); + scope.host = boot && /^[0-9a-f-]{36}$/.test(boot) && (scope.network || scope.container) + ? scopeDigest("linux", hostname(), boot) + : scopeDigest("unverifiable", randomUUID()); + return scope; +} + +/** + * Whether a recorded owner scope proves the record was written where this process runs. Only an + * exact, well-formed match counts; anything else (absent, malformed, partial) is "no". + */ +export function sameRuntimeOwnerScope(recorded: unknown, own: RuntimeOwnerScope): boolean { + if (!recorded || typeof recorded !== "object" || Array.isArray(recorded)) return false; + const record = recorded as Record; + if (typeof record.host !== "string" || record.host !== own.host) return false; + if (own.network === undefined && own.container === undefined) { + return record.network === undefined && record.container === undefined; + } + return (own.network !== undefined && record.network === own.network) + || (own.container !== undefined && record.container === own.container); } export function writePid(pid: number): void { @@ -76,11 +141,12 @@ export type RuntimePortState = { */ siblingOfPort?: number; /** - * {@link runtimeHostIdentity} of the writer, stamped by {@link writeRuntimePort}. Not validated - * here: a malformed value must not hide a live owner from status, stop or liveness, and the one - * reader that uses it (`observeProxyLiveness`) accepts only an exact match. + * {@link runtimeOwnerScope} of the owner, stamped by {@link writeRuntimePort} when this process + * writes its own record. Not validated here: a malformed value must not hide a live owner from + * status, stop or liveness, and the one reader that uses it (`observeProxyLiveness`) accepts + * only an exact match ({@link sameRuntimeOwnerScope}). */ - hostIdentity?: string; + ownerScope?: RuntimeOwnerScope; }; export function isValidRuntimePortState(value: unknown): value is RuntimePortState { @@ -104,7 +170,11 @@ export function isValidRuntimePortState(value: unknown): value is RuntimePortSta export function writeRuntimePort(state: RuntimePortState): void { ensureProcessStateDir(); publishedOwnershipRecords = true; - atomicWriteFile(getRuntimePortPath(), JSON.stringify({ ...state, hostIdentity: runtimeHostIdentity() }, null, 2) + "\n"); + // Only the owner can say where it runs; a record written on another process's behalf keeps + // whatever identity it already carried. + const ownerScope = state.pid === process.pid ? runtimeOwnerScope() : state.ownerScope; + const record = { ...state, ...(ownerScope === undefined ? {} : { ownerScope }) }; + atomicWriteFile(getRuntimePortPath(), JSON.stringify(record, null, 2) + "\n"); // The record proves this home's owner only to a reader that knows where it lives. // One pointer in the shared registry makes the record findable from every home; // it is best-effort because ownership never depends on the registry write landing. diff --git a/src/server/proxy-liveness.ts b/src/server/proxy-liveness.ts index 0da0ca1d7dd..f47d3973122 100644 --- a/src/server/proxy-liveness.ts +++ b/src/server/proxy-liveness.ts @@ -14,7 +14,7 @@ import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target import { normalizeBindHostname } from "./bind-host"; import { hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, - runtimeHostIdentity, verifyPidIdentity, + runtimeOwnerScope, sameRuntimeOwnerScope, verifyPidIdentity, type RuntimeOwnerScope, } from "../config/process-state"; import { createServer, isIP } from "node:net"; import { loopbackFamilyAvailability } from "../lib/loopback-family"; @@ -96,7 +96,7 @@ export interface LivenessIo { * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget. */ bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise; - readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; hostIdentity?: string } | null; + readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; ownerScope?: unknown } | null; /** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */ ownProcessFn?: () => OwnProcess; configFn?: () => { port?: number; hostname?: string }; @@ -522,31 +522,32 @@ export interface OwnProcess { pid: number; /** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */ publishedOwnership: boolean; - hostIdentity: string; + scope: RuntimeOwnerScope; } function currentOwnProcess(): OwnProcess { - return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), hostIdentity: runtimeHostIdentity() }; + return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), scope: runtimeOwnerScope() }; } /** Recovery additionally covers both available loopback families and fresh PID identity. */ export async function observeProxyLiveness(io: LivenessIo = {}): Promise { - let alivePid: number | null = null, runtimePid: number | undefined, runtimeHost: unknown, unknown = false; + let alivePid: number | null = null, runtimePid: number | undefined, runtimeScope: unknown, unknown = false; const live = await discoverLiveProxy(io, { pid: pid => { alivePid = pid; }, - runtimePid: (pid, hostIdentity) => { runtimePid = pid; runtimeHost = hostIdentity; }, + runtimePid: (pid, ownerScope) => { runtimePid = pid; runtimeScope = ownerScope; }, endpoint: result => { if (result === "unknown") unknown = true; }, }); if (live) return { kind: "live", live }; // #8552: in a container the proxy is PID 1 on every start, so the record the previous container // left names this very process; signalling it succeeds and its command line is ocx. Such a // record is stale only when this process never published ownership itself (so the writer was an - // earlier lifetime of the PID number) AND the runtime record was written under this host name. - // The host check is what keeps a RollingUpdate safe: the old pod is also PID 1 and its listener - // sits in another network namespace, but its pod name differs. A record without the identity - // (an older version, which may still be serving during an upgrade) keeps the refusal. + // earlier lifetime of the PID number) AND the runtime record's owner ran where this process runs + // (same host and boot, plus the same network namespace or container). That scope check keeps a + // RollingUpdate safe: the old pod is also PID 1 and its listener sits in another network + // namespace, which the endpoint probe cannot see. A record without a scope (an older version, + // which may still be serving during an upgrade) keeps the refusal. const own = (io.ownProcessFn ?? currentOwnProcess)(); - const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && runtimeHost === own.hostIdentity; + const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && sameRuntimeOwnerScope(runtimeScope, own.scope); const candidates = new Set(); if (alivePid !== null) candidates.add(alivePid); if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid); @@ -583,7 +584,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise void; - runtimePid: (pid: number | undefined, hostIdentity: unknown) => void; + runtimePid: (pid: number | undefined, ownerScope: unknown) => void; endpoint: (result: EndpointLiveness) => void; }): Promise { // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too @@ -685,7 +686,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: { // sibling command, manual deletion). The runtime record still says where it listens — // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it. const record = readRuntimeFn(); - observer?.runtimePid(record?.pid, record?.hostIdentity); + observer?.runtimePid(record?.pid, record?.ownerScope); if (record?.port && record.port !== probedPort) { if (budgetExhausted()) return expired(); const expectedPid = typeof record.pid === "number" ? record.pid : undefined; diff --git a/structure/runtime.md b/structure/runtime.md index 571beb40a77..e2b170771f6 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the starting process's own PID, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with a `hostIdentity` equal to this host's (`runtimeHostIdentity`, a SHA-256 prefix of the host name that `writeRuntimePort` stamps). A restarted container keeps its pod's host name; a replacement pod, which can start while the old pod still serves the same volume from another network namespace, does not, so its refusal stands. A record without the identity (an older version), a PID file without a runtime record, and a malformed identity keep the refusal; the identity is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the starting process's own PID, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name and boot id plus the same network namespace (a container restarted inside its Kubernetes pod) or the same Docker/Podman container id from mountinfo (a restarted container; one id never runs twice at once); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, another node, or a WSL distro sharing a Windows home matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/config.test.ts b/tests/server/config.test.ts index e30fce6dc04..f9a4c4915fa 100644 --- a/tests/server/config.test.ts +++ b/tests/server/config.test.ts @@ -35,7 +35,6 @@ import { import * as windowsAcl from "../../src/lib/windows-secret-acl"; import { setTrustedWindowsSystemDirectoryResolverForTests } from "../../src/lib/windows-elevation"; import { AtomicWriteResidualTempError, atomicWriteFile, atomicWriteFileAsync, hardenConfigDir, hardenExistingSecret, renameAtomicFile, saveConfig } from "../../src/config"; -import { runtimeHostIdentity } from "../../src/config/process-state"; import { nextAtomicTempSequence } from "../../src/config/atomic-write"; import { flushConfigDirHardeningForTests } from "../../src/config/paths"; import { @@ -3123,8 +3122,8 @@ describe("opencodex config defaults", () => { const attestationSecret = "A".repeat(43); writeRuntimePort({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); - expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); + expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); expect(readRuntimePort(9999)).toBeNull(); }); diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts index c2bf83a96f7..b4c1cc50b29 100644 --- a/tests/server/proxy-liveness-self-pid-record.test.ts +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { runtimeHostIdentity } from "../../src/config/process-state"; +import { containerIdFromMountinfo, runtimeOwnerScope, sameRuntimeOwnerScope } from "../../src/config/process-state"; import { observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; import { watchdogMs } from "../helpers/ci-watchdog"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -30,38 +30,57 @@ function ioFor(extra: Partial): LivenessIo { }; } -const HOST = "0123456789abcdef0123456789abcdef"; -const unpublished = () => ({ pid: process.pid, publishedOwnership: false, hostIdentity: HOST }); -const selfRecord = (hostIdentity: string | null = HOST) => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", ...(hostIdentity === null ? {} : { hostIdentity }) }); +const OWN = { host: "0123456789abcdef0123456789abcdef", network: "11111111111111111111111111111111" }; +const unpublished = () => ({ pid: process.pid, publishedOwnership: false, scope: OWN }); +const selfRecord = (ownerScope: unknown = OWN) => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", ...(ownerScope === null ? {} : { ownerScope }) }); describe("a record naming this process's own PID", () => { - test("a runtime record this host's earlier lifetime of this PID left is absent", async () => { + test("a runtime record an earlier run of this container left is absent", async () => { const io = ioFor({ readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); }); + test("a restarted Docker container matches on its container id when the network namespace is new", async () => { + const own = { host: OWN.host, network: "22222222222222222222222222222222", container: "33333333333333333333333333333333" }; + const io = ioFor({ + readRuntimeFn: () => selfRecord({ host: OWN.host, network: OWN.network, container: own.container }), + ownProcessFn: () => ({ pid: process.pid, publishedOwnership: false, scope: own }), + }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); + }); + test("a pid file beside that runtime record is absent too", async () => { const io = ioFor({ readPidFn: () => process.pid, readRuntimeFn: () => selfRecord(), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); }); - test("a pid file alone carries no host identity and still refuses", async () => { + test("a pid file alone carries no owner scope and still refuses", async () => { const io = ioFor({ readPidFn: () => process.pid, ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); - // Every container is PID 1: during a RollingUpdate the old pod still serves this state volume - // from another network namespace, and only its host name tells it apart. - test("a record another host wrote under the same PID number still refuses", async () => { - const io = ioFor({ readRuntimeFn: () => selfRecord("fedcba9876543210fedcba9876543210"), ownProcessFn: unpublished }); + // Every container is PID 1. During a RollingUpdate, or with a fixed hostname and start-first, the + // old container still serves this volume from another network namespace the probe cannot see. + test("a same-host record from another network namespace and container still refuses", async () => { + const io = ioFor({ + readRuntimeFn: () => selfRecord({ host: OWN.host, network: "44444444444444444444444444444444", container: "55555555555555555555555555555555" }), + ownProcessFn: unpublished, + }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); - test("a record without a host identity (an older version, maybe still serving) still refuses", async () => { - const io = ioFor({ readRuntimeFn: () => selfRecord(null), ownProcessFn: unpublished }); + test("a record from another host or boot still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord({ ...OWN, host: "fedcba9876543210fedcba9876543210" }), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); + for (const [label, scope] of [["no owner scope (an older version, maybe still serving)", null], ["a malformed owner scope", OWN.host], ["a host-only scope on Linux", { host: OWN.host }]] as const) { + test(`a record with ${label} still refuses`, async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord(scope), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + } + test("once this process has published ownership, its own PID still reads as a live owner", async () => { const io = ioFor({ readRuntimeFn: () => selfRecord(), @@ -89,6 +108,33 @@ describe("a record naming this process's own PID", () => { }); }); +describe("owner scope comparison", () => { + test("Linux needs the host plus the network namespace or the container", () => { + const own = { host: "h", network: "n", container: "c" }; + expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", network: "x", container: "c" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", network: "x", container: "y" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "other", network: "n", container: "c" }, own)).toBe(false); + for (const malformed of [undefined, null, "h", ["h"], { host: 1, network: "n" }]) expect(sameRuntimeOwnerScope(malformed, own)).toBe(false); + }); + + test("a machine-wide PID space compares the host alone, and never against a Linux record", () => { + const own = { host: "h" }; + expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "other" }, own)).toBe(false); + }); + + test("container ids come from the files Docker and Podman bind-mount, not from pod sandboxes", () => { + const id = "a".repeat(64); + expect(containerIdFromMountinfo(`1234 1200 0:52 /var/lib/docker/containers/${id}/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw\n`)).toBe(id); + expect(containerIdFromMountinfo(`1234 1200 0:52 /containers/storage/overlay-containers/${id}/userdata/resolv.conf /etc/resolv.conf rw - tmpfs tmpfs rw\n`)).toBe(id); + expect(containerIdFromMountinfo(`1234 1200 0:52 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/${id}/hostname /etc/hostname rw - ext4 /dev/sda1 rw\n`)).toBeUndefined(); + expect(containerIdFromMountinfo("25 1 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n")).toBeUndefined(); + }); +}); + const roots: string[] = []; const children: Array> = []; afterEach(async () => { @@ -111,7 +157,7 @@ const START_WAIT_MS = watchdogMs(15_000); // `exec` keeps the shell's PID, so the shell plays the previous container and `ocx start` the next // one: same PID, new lifetime, records naming it already on disk. Windows has no exec handoff. -async function startAfterOwnLifetime(hostIdentity: string): Promise<{ child: ReturnType; port: number; ocxHome: string; served: boolean; stderr: Promise }> { +async function startAfterOwnLifetime(ownerScope: unknown): Promise<{ child: ReturnType; port: number; ocxHome: string; served: boolean; stderr: Promise }> { const root = mkdtempSync(join(tmpdir(), "ocx-self-pid-")); roots.push(root); const ocxHome = join(root, "ocx"), codexHome = join(root, "codex"), home = join(root, "home"), runtime = join(root, "runtime"); @@ -129,7 +175,7 @@ async function startAfterOwnLifetime(hostIdentity: string): Promise<{ child: Ret })); const script = [ 'printf %s "$$" > "$OPENCODEX_HOME/ocx.pid"', - 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1","hostIdentity":"%s"}\' "$$" "$SELF_PID_PORT" "$SELF_PID_HOST" > "$OPENCODEX_HOME/runtime-port.json"', + 'printf \'{"pid":%s,"port":%s,"hostname":"127.0.0.1","ownerScope":%s}\' "$$" "$SELF_PID_PORT" "$SELF_PID_SCOPE" > "$OPENCODEX_HOME/runtime-port.json"', 'exec "$SELF_PID_BUN" "$SELF_PID_CLI" start --port "$SELF_PID_PORT"', ].join(" && "); const child = Bun.spawn(["/bin/sh", "-c", script], { @@ -145,7 +191,7 @@ async function startAfterOwnLifetime(hostIdentity: string): Promise<{ child: Ret SELF_PID_BUN: process.execPath, SELF_PID_CLI: repoPath("src/cli/index.ts"), SELF_PID_PORT: String(port), - SELF_PID_HOST: hostIdentity, + SELF_PID_SCOPE: JSON.stringify(ownerScope), }, stdout: "pipe", stderr: "pipe", @@ -167,8 +213,8 @@ async function startAfterOwnLifetime(hostIdentity: string): Promise<{ child: Ret const posix = process.platform !== "win32"; -test.skipIf(!posix)("ocx start recovers records this host's earlier lifetime of its PID left", async () => { - const { child, port, ocxHome, served, stderr } = await startAfterOwnLifetime(runtimeHostIdentity()); +test.skipIf(!posix)("ocx start recovers records an earlier run of its PID left in the same place", async () => { + const { child, port, ocxHome, served, stderr } = await startAfterOwnLifetime(runtimeOwnerScope()); if (!served) { if (child.exitCode === null) child.kill(); await child.exited; @@ -176,16 +222,16 @@ test.skipIf(!posix)("ocx start recovers records this host's earlier lifetime of const reason = text.split("\n").find(line => line.startsWith("error:")) ?? text.trim().slice(0, 400); throw new Error(`start did not serve as PID ${child.pid}: exit ${child.exitCode}: ${reason}`); } - const record = JSON.parse(readFileSync(join(ocxHome, "runtime-port.json"), "utf8")) as { pid: number; port: number; attestationSecret?: string; hostIdentity?: string }; + const record = JSON.parse(readFileSync(join(ocxHome, "runtime-port.json"), "utf8")) as { pid: number; port: number; attestationSecret?: string; ownerScope?: unknown }; expect(record.pid).toBe(child.pid); expect(record.port).toBe(port); // A fresh publication: the stale record had no attestation secret. expect(typeof record.attestationSecret).toBe("string"); - expect(record.hostIdentity).toBe(runtimeHostIdentity()); + expect(record.ownerScope).toEqual(runtimeOwnerScope()); }, Math.max(30_000, START_WAIT_MS * 2)); -test.skipIf(!posix)("ocx start still refuses a same-PID record another host wrote", async () => { - const { child, served, stderr, ocxHome } = await startAfterOwnLifetime("fedcba9876543210fedcba9876543210"); +test.skipIf(!posix)("ocx start still refuses a same-PID record written somewhere else", async () => { + const { child, served, stderr, ocxHome } = await startAfterOwnLifetime({ ...runtimeOwnerScope(), host: "fedcba9876543210fedcba9876543210" }); expect(served).toBe(false); expect(await child.exited).toBe(1); expect(await stderr).toContain("Proxy ownership is unverified"); diff --git a/tests/service/process-state.test.ts b/tests/service/process-state.test.ts index 7b262824aa7..d2669745cd7 100644 --- a/tests/service/process-state.test.ts +++ b/tests/service/process-state.test.ts @@ -15,7 +15,7 @@ import { readRuntimePort, removePid, removeRuntimePort, - runtimeHostIdentity, + runtimeOwnerScope, setOcxStartProcessCacheForTests, setProcessCommandLineExecForTests, setProcessCommandLineNativeForTests, @@ -389,11 +389,25 @@ describe("proxy process-state ownership", () => { const attestationSecret = "A".repeat(43); writeRuntimePort({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); - expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); - expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret, hostIdentity: runtimeHostIdentity() }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); + expect(readRuntimePort(1234)).toEqual({ pid: 1234, port: 58195, hostname: "0.0.0.0", attestationSecret }); expect(readRuntimePort(9999)).toBeNull(); }); + // #8552: the scope says where the OWNER runs, so only the owner may stamp it. + test("a runtime record this process writes for itself carries its owner scope; one for another PID keeps its own", () => { + const scope = runtimeOwnerScope(); + expect(scope.host).toMatch(/^[0-9a-f]{32}$/); + expect(runtimeOwnerScope()).toBe(scope); + if (process.platform !== "linux") expect(scope).toEqual({ host: scope.host }); + writeRuntimePort({ pid: process.pid, port: 58195 }); + expect(readRuntimePort()?.ownerScope).toEqual(scope); + writeRuntimePort({ pid: 1234, port: 58195, ownerScope: { host: "kept-as-written" } }); + expect(readRuntimePort()?.ownerScope).toEqual({ host: "kept-as-written" }); + writeRuntimePort({ pid: 1234, port: 58195 }); + expect(readRuntimePort()?.ownerScope).toBeUndefined(); + }); + test("runtime port removal preserves newer pid state", () => { writeRuntimePort({ pid: 1234, port: 58195 }); removeRuntimePort(9999); @@ -414,12 +428,12 @@ describe("proxy process-state ownership", () => { expect(readRuntimePort()).toBeNull(); }); - test("a sibling record carries the live owner's port; every other record omits the field", () => { - // Absent means "not a sibling", and the writer must not add the key. Every record carries the - // writer's host identity (#8552). + test("a sibling record carries the live owner's port; every other record keeps its bytes", () => { + // Absent means "not a sibling", and the writer must not add the key: a non-sibling record is + // byte-identical to the one written before the field existed. writeRuntimePort({ pid: 1234, port: 58195, hostname: "127.0.0.1", ...siblingRuntimeField() }); expect(readFileSync(getRuntimePortPath(), "utf-8")) - .toBe(`${JSON.stringify({ pid: 1234, port: 58195, hostname: "127.0.0.1", hostIdentity: runtimeHostIdentity() }, null, 2)}\n`); + .toBe(`${JSON.stringify({ pid: 1234, port: 58195, hostname: "127.0.0.1" }, null, 2)}\n`); expect(readRuntimePort()?.siblingOfPort).toBeUndefined(); markSiblingStart(10100); @@ -428,7 +442,7 @@ describe("proxy process-state ownership", () => { } finally { resetSiblingStartForTests(); } - expect(readRuntimePort()).toEqual({ pid: 1234, port: 10199, hostname: "127.0.0.1", siblingOfPort: 10100, hostIdentity: runtimeHostIdentity() }); + expect(readRuntimePort()).toEqual({ pid: 1234, port: 10199, hostname: "127.0.0.1", siblingOfPort: 10100 }); for (const siblingOfPort of [0, 70000, 1.5, "10100", null]) { writeFileSync(getRuntimePortPath(), JSON.stringify({ pid: 1234, port: 10199, siblingOfPort }), "utf-8"); From eccd2cc52a2e74a451e225caf8c938b3f05c5c25 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 12 Nov 2026 01:33:47 +0900 Subject: [PATCH 4/5] fix(start): key the container scope on the root overlay layer and keep the network leg within one boot --- .../src/content/docs/guides/remote-hub.md | 15 ++-- src/config/process-state.ts | 68 ++++++++++++------- structure/runtime.md | 2 +- .../proxy-liveness-self-pid-record.test.ts | 53 ++++++++++----- 4 files changed, 89 insertions(+), 49 deletions(-) diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index ed2d2b648ed..a97d29c4781 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -750,17 +750,20 @@ the authenticated routed-response check above for your deployment. A container runs the proxy as PID 1 on every start. After an unclean exit (an OOM kill, a crash, a node failure) the state volume can still hold `ocx.pid` and `runtime-port.json` naming PID 1, and the next start has to decide whether that owner is gone. It recovers on its own when the record was -written by the same container in an earlier run: a container restarted in place (`docker restart` -of the same container, or a Kubernetes container restart inside its pod) on the same machine, by -an OpenCodex version that records where its owner runs. +written by the same container in an earlier run, by an OpenCodex version that records where its +owner runs: the same Docker or Podman container started again (`docker restart`, a restart policy, +including after a host reboot) when its root filesystem is an overlay (the default `overlay2` +driver), or a Kubernetes container restarted inside its pod while the node stays up. Every other case refuses with `Proxy ownership is unverified; startup refused`, because the old owner could still be serving the same volume from another container: a replacement Kubernetes pod, -a recreated Compose container, another node, or a record written by an older version (the first -restart after an upgrade). Confirm that nothing else serves this state directory, then move +a recreated Compose container, a pod that comes back after a node reboot, another node, or a record +written by an older version (the first restart after an upgrade). Confirm that nothing else serves this state directory, then move `runtime-port.json` and `ocx.pid` aside once and start again. Run a single writer per state volume: a Kubernetes `Deployment` that shares a volume should use `strategy: Recreate` (or -`maxSurge: 0`) so the old pod stops before the new one starts. +`maxSurge: 0`) so the old pod stops before the new one starts. Do not share one state volume between +processes in one network namespace (host networking, sidecars, several WSL distros), or between +sandboxes restored from one VM snapshot. ## Rollback diff --git a/src/config/process-state.ts b/src/config/process-state.ts index ac6b15257b0..c47b72e3eb9 100644 --- a/src/config/process-state.ts +++ b/src/config/process-state.ts @@ -47,22 +47,29 @@ export function hasPublishedOwnershipRecords(): boolean { * Where the owner of a runtime record ran, as digests that store none of their inputs (#8552). * * Every container runs the proxy as PID 1, so a PID alone cannot say whether a record came from - * this container's previous run or from another container serving the same state volume. On - * Linux, `host` covers the host name and boot id, and at least one of two stronger facts must - * also match ({@link sameRuntimeOwnerScope}): + * this container's previous run or from another container serving the same state volume. The + * `host` name must match, and on Linux at least one of two stronger facts as well + * ({@link sameRuntimeOwnerScope}): * - * - `network`, the network namespace: a container restarted inside its Kubernetes pod keeps it, - * because the pod sandbox owns it; - * - `container`, the Docker/Podman container (or CRI-O pod sandbox) id read from mountinfo: a - * restarted container keeps it, and one id never runs twice at the same time. + * - `network` with `boot`, the network namespace within one boot: a container restarted inside + * its Kubernetes pod keeps it, because the pod sandbox owns it. Namespace inode numbers repeat + * across machines and boots, so the boot id has to match too; + * - `container`, the writable layer of the overlay mounted at `/` (its `upperdir`): a Docker or + * Podman container started again keeps it, across a host reboot as well, and no other container + * ever mounts it, not even one sharing its network (`--network container:`) or its pod. A + * container id is NOT used: a container joined to another's network sees that one's id in + * mountinfo. * - * A replacement pod, a second container with a fixed `hostname`, another node, or a WSL distro - * sharing a Windows home matches neither. Elsewhere the PID space is the whole machine and the host - * name suffices. Linux evidence that cannot be read yields a per-process value that matches no - * other lifetime, so the self-record proof fails closed. + * A replacement pod, a second container with a fixed `hostname`, or another node matches neither. + * Processes that share one network namespace (host networking, pod sidecars, WSL distros, which + * share one VM) can match on `network`; for them the loopback probe still finds a listening owner. + * Elsewhere the PID space is the whole machine and the host name suffices. Linux evidence that + * cannot be read yields a per-process value that matches no other lifetime, so the self-record + * proof fails closed. */ export interface RuntimeOwnerScope { host: string; + boot?: string; network?: string; container?: string; } @@ -78,11 +85,23 @@ function scopeDigest(...parts: string[]): string { return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32); } -const MOUNTINFO_CONTAINER_ID = /\/(?:containers|overlay-containers)\/([0-9a-f]{64})\/(?:userdata\/)?(?:resolv\.conf|hostname|hosts)(?:\s|$)/m; - -/** The container id Docker or Podman bind-mounts `/etc/hostname` and friends from, if any. */ -export function containerIdFromMountinfo(mountinfo: string): string | undefined { - return MOUNTINFO_CONTAINER_ID.exec(mountinfo)?.[1]; +/** + * The `upperdir` of the overlay mounted at `/`, if the root filesystem is one: a container's own + * writable layer. The last root mount wins, as it is the one visible. Any other root filesystem + * (bare host, btrfs/zfs/vfs drivers, fuse-overlayfs) yields nothing. + */ +export function rootOverlayUpperdirFromMountinfo(mountinfo: string): string | undefined { + let upperdir: string | undefined; + for (const line of mountinfo.split("\n")) { + const separator = line.indexOf(" - "); + if (separator < 0) continue; + const fields = line.slice(0, separator).split(" "); + if (fields[4] !== "/") continue; + const [fstype, , superOptions = ""] = line.slice(separator + 3).split(" "); + const match = fstype === "overlay" ? /(?:^|,)upperdir=([^,]+)/.exec(superOptions) : null; + upperdir = match?.[1]; + } + return upperdir; } function readOptional(read: () => string): string | undefined { @@ -93,13 +112,14 @@ function computeRuntimeOwnerScope(): RuntimeOwnerScope { if (process.platform !== "linux") return { host: scopeDigest(process.platform, hostname()) }; const boot = readOptional(() => readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()); const network = readOptional(() => readlinkSync("/proc/self/ns/net")); - const container = readOptional(() => containerIdFromMountinfo(readFileSync("/proc/self/mountinfo", "utf8")) ?? ""); - const scope: RuntimeOwnerScope = { host: "" }; - if (network && /^net:\[\d+\]$/.test(network)) scope.network = scopeDigest("net", network); + const container = readOptional(() => rootOverlayUpperdirFromMountinfo(readFileSync("/proc/self/mountinfo", "utf8")) ?? ""); + const scope: RuntimeOwnerScope = { host: scopeDigest("linux", hostname()) }; + if (boot && /^[0-9a-f-]{36}$/.test(boot) && network && /^net:\[\d+\]$/.test(network)) { + scope.boot = scopeDigest("boot", boot); + scope.network = scopeDigest("net", network); + } if (container) scope.container = scopeDigest("container", container); - scope.host = boot && /^[0-9a-f-]{36}$/.test(boot) && (scope.network || scope.container) - ? scopeDigest("linux", hostname(), boot) - : scopeDigest("unverifiable", randomUUID()); + if (!scope.network && !scope.container) return { host: scopeDigest("unverifiable", randomUUID()) }; return scope; } @@ -112,9 +132,9 @@ export function sameRuntimeOwnerScope(recorded: unknown, own: RuntimeOwnerScope) const record = recorded as Record; if (typeof record.host !== "string" || record.host !== own.host) return false; if (own.network === undefined && own.container === undefined) { - return record.network === undefined && record.container === undefined; + return record.boot === undefined && record.network === undefined && record.container === undefined; } - return (own.network !== undefined && record.network === own.network) + return (own.network !== undefined && record.network === own.network && own.boot !== undefined && record.boot === own.boot) || (own.container !== undefined && record.container === own.container); } diff --git a/structure/runtime.md b/structure/runtime.md index e2b170771f6..9ba1096b663 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the starting process's own PID, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name and boot id plus the same network namespace (a container restarted inside its Kubernetes pod) or the same Docker/Podman container id from mountinfo (a restarted container; one id never runs twice at once); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, another node, or a WSL distro sharing a Windows home matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the PID of the process running the recovery recheck, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name plus either the same network namespace within the same boot (a container restarted inside its Kubernetes pod; namespace inode numbers repeat across boots) or the same writable layer of the overlay mounted at `/` (its `upperdir` in mountinfo: a Docker or Podman container started again, also across a host reboot; no other container mounts it, while a container id is visible to every container joined to that container's network and is therefore not used); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, or another node matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. Processes sharing one network namespace (host networking, pod sidecars, WSL distros in one VM) can match, and there the loopback probe still finds a listening owner; an owner that is alive but no longer listening, an owner exit cleanup that removes records by PID only, and sandboxes that fake namespace numbers or clone a whole VM (gVisor, restored Firecracker/Kata snapshots) are residuals for a shared state volume. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts index b4c1cc50b29..87fb332eba4 100644 --- a/tests/server/proxy-liveness-self-pid-record.test.ts +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { containerIdFromMountinfo, runtimeOwnerScope, sameRuntimeOwnerScope } from "../../src/config/process-state"; +import { rootOverlayUpperdirFromMountinfo, runtimeOwnerScope, sameRuntimeOwnerScope } from "../../src/config/process-state"; import { observeProxyLiveness, type LivenessIo } from "../../src/server/proxy-liveness"; import { watchdogMs } from "../helpers/ci-watchdog"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -30,7 +30,7 @@ function ioFor(extra: Partial): LivenessIo { }; } -const OWN = { host: "0123456789abcdef0123456789abcdef", network: "11111111111111111111111111111111" }; +const OWN = { host: "0123456789abcdef0123456789abcdef", boot: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", network: "11111111111111111111111111111111" }; const unpublished = () => ({ pid: process.pid, publishedOwnership: false, scope: OWN }); const selfRecord = (ownerScope: unknown = OWN) => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", ...(ownerScope === null ? {} : { ownerScope }) }); @@ -40,10 +40,12 @@ describe("a record naming this process's own PID", () => { expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); }); - test("a restarted Docker container matches on its container id when the network namespace is new", async () => { - const own = { host: OWN.host, network: "22222222222222222222222222222222", container: "33333333333333333333333333333333" }; + // Docker gives a restarted container a new network namespace, and a host reboot a new boot id, + // but the container keeps its own writable layer. + test("a restarted Docker container matches on its writable layer, even across a host reboot", async () => { + const own = { host: OWN.host, boot: "cccccccccccccccccccccccccccccccc", network: "22222222222222222222222222222222", container: "33333333333333333333333333333333" }; const io = ioFor({ - readRuntimeFn: () => selfRecord({ host: OWN.host, network: OWN.network, container: own.container }), + readRuntimeFn: () => selfRecord({ host: OWN.host, boot: OWN.boot, network: OWN.network, container: own.container }), ownProcessFn: () => ({ pid: process.pid, publishedOwnership: false, scope: own }), }); expect(await observeProxyLiveness(io)).toEqual({ kind: "absent", live: null }); @@ -69,11 +71,17 @@ describe("a record naming this process's own PID", () => { expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); - test("a record from another host or boot still refuses", async () => { + test("a record from another host still refuses", async () => { const io = ioFor({ readRuntimeFn: () => selfRecord({ ...OWN, host: "fedcba9876543210fedcba9876543210" }), ownProcessFn: unpublished }); expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); }); + // Namespace inode numbers repeat across boots and machines. + test("an equal network namespace from another boot still refuses", async () => { + const io = ioFor({ readRuntimeFn: () => selfRecord({ ...OWN, boot: "dddddddddddddddddddddddddddddddd" }), ownProcessFn: unpublished }); + expect(await observeProxyLiveness(io)).toEqual({ kind: "unknown", live: null }); + }); + for (const [label, scope] of [["no owner scope (an older version, maybe still serving)", null], ["a malformed owner scope", OWN.host], ["a host-only scope on Linux", { host: OWN.host }]] as const) { test(`a record with ${label} still refuses`, async () => { const io = ioFor({ readRuntimeFn: () => selfRecord(scope), ownProcessFn: unpublished }); @@ -109,13 +117,15 @@ describe("a record naming this process's own PID", () => { }); describe("owner scope comparison", () => { - test("Linux needs the host plus the network namespace or the container", () => { - const own = { host: "h", network: "n", container: "c" }; - expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(true); - expect(sameRuntimeOwnerScope({ host: "h", network: "x", container: "c" }, own)).toBe(true); - expect(sameRuntimeOwnerScope({ host: "h", network: "x", container: "y" }, own)).toBe(false); + test("Linux needs the host plus the network namespace within one boot, or the container", () => { + const own = { host: "h", boot: "b", network: "n", container: "c" }; + expect(sameRuntimeOwnerScope({ host: "h", boot: "b", network: "n" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", boot: "x", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", boot: "x", network: "x", container: "c" }, own)).toBe(true); + expect(sameRuntimeOwnerScope({ host: "h", boot: "b", network: "x", container: "y" }, own)).toBe(false); expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(false); - expect(sameRuntimeOwnerScope({ host: "other", network: "n", container: "c" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "other", boot: "b", network: "n", container: "c" }, own)).toBe(false); for (const malformed of [undefined, null, "h", ["h"], { host: 1, network: "n" }]) expect(sameRuntimeOwnerScope(malformed, own)).toBe(false); }); @@ -123,15 +133,22 @@ describe("owner scope comparison", () => { const own = { host: "h" }; expect(sameRuntimeOwnerScope({ host: "h" }, own)).toBe(true); expect(sameRuntimeOwnerScope({ host: "h", network: "n" }, own)).toBe(false); + expect(sameRuntimeOwnerScope({ host: "h", boot: "b" }, own)).toBe(false); expect(sameRuntimeOwnerScope({ host: "other" }, own)).toBe(false); }); - test("container ids come from the files Docker and Podman bind-mount, not from pod sandboxes", () => { - const id = "a".repeat(64); - expect(containerIdFromMountinfo(`1234 1200 0:52 /var/lib/docker/containers/${id}/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw\n`)).toBe(id); - expect(containerIdFromMountinfo(`1234 1200 0:52 /containers/storage/overlay-containers/${id}/userdata/resolv.conf /etc/resolv.conf rw - tmpfs tmpfs rw\n`)).toBe(id); - expect(containerIdFromMountinfo(`1234 1200 0:52 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/${id}/hostname /etc/hostname rw - ext4 /dev/sda1 rw\n`)).toBeUndefined(); - expect(containerIdFromMountinfo("25 1 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n")).toBeUndefined(); + test("the container key is the root overlay's own writable layer, never a shared id", () => { + const docker = "/var/lib/docker/overlay2/" + "a".repeat(64) + "/diff"; + const root = (fstype: string, options: string) => `1001 1000 0:52 / / rw,relatime master:1 - ${fstype} ${fstype} ${options}`; + // A container joined to another's network bind-mounts that one's hostname; it must not count. + const joined = `1201 1001 8:1 /var/lib/docker/containers/${"b".repeat(64)}/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw`; + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `ro,lowerdir=/l1:/l2,upperdir=${docker},workdir=/w`), joined].join("\n") + "\n")).toBe(docker); + expect(rootOverlayUpperdirFromMountinfo(joined + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("ext4", "rw,errors=remount-ro") + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("fuse.fuse-overlayfs", "rw,user_id=0") + "\n")).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l1") + "\n")).toBeUndefined(); + // The visible root is the last one mounted. + expect(rootOverlayUpperdirFromMountinfo([root("overlay", "rw,upperdir=/old,workdir=/w"), root("overlay", "rw,upperdir=/new,workdir=/w")].join("\n"))).toBe("/new"); }); }); From 97310494f03bdf77814f7758722daff518bc7707 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 12 Nov 2026 01:39:51 +0900 Subject: [PATCH 5/5] fix(start): count only randomly named overlay layers as the container scope --- .../src/content/docs/guides/remote-hub.md | 11 +++++----- src/config/process-state.ts | 22 +++++++++++-------- structure/runtime.md | 2 +- .../proxy-liveness-self-pid-record.test.ts | 12 ++++++++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index a97d29c4781..cee0ea5a84d 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -751,9 +751,10 @@ A container runs the proxy as PID 1 on every start. After an unclean exit (an OO node failure) the state volume can still hold `ocx.pid` and `runtime-port.json` naming PID 1, and the next start has to decide whether that owner is gone. It recovers on its own when the record was written by the same container in an earlier run, by an OpenCodex version that records where its -owner runs: the same Docker or Podman container started again (`docker restart`, a restart policy, -including after a host reboot) when its root filesystem is an overlay (the default `overlay2` -driver), or a Kubernetes container restarted inside its pod while the node stays up. +owner runs: a Kubernetes container restarted inside its pod while the node stays up, or the same +Docker or Podman container started again (`docker restart`, a restart policy, including after a +host reboot) when its writable layer has a random id, as with Docker's `overlay2` storage driver +and Podman. Docker's containerd image store numbers its layers instead, so a restart there refuses. Every other case refuses with `Proxy ownership is unverified; startup refused`, because the old owner could still be serving the same volume from another container: a replacement Kubernetes pod, @@ -762,8 +763,8 @@ written by an older version (the first restart after an upgrade). Confirm that n `runtime-port.json` and `ocx.pid` aside once and start again. Run a single writer per state volume: a Kubernetes `Deployment` that shares a volume should use `strategy: Recreate` (or `maxSurge: 0`) so the old pod stops before the new one starts. Do not share one state volume between -processes in one network namespace (host networking, sidecars, several WSL distros), or between -sandboxes restored from one VM snapshot. +processes in one network namespace (host networking, sidecars, several WSL distros), between +sandboxes restored from one VM snapshot, or between machines with the same host name. ## Rollback diff --git a/src/config/process-state.ts b/src/config/process-state.ts index c47b72e3eb9..a80bdefa035 100644 --- a/src/config/process-state.ts +++ b/src/config/process-state.ts @@ -54,11 +54,14 @@ export function hasPublishedOwnershipRecords(): boolean { * - `network` with `boot`, the network namespace within one boot: a container restarted inside * its Kubernetes pod keeps it, because the pod sandbox owns it. Namespace inode numbers repeat * across machines and boots, so the boot id has to match too; - * - `container`, the writable layer of the overlay mounted at `/` (its `upperdir`): a Docker or - * Podman container started again keeps it, across a host reboot as well, and no other container - * ever mounts it, not even one sharing its network (`--network container:`) or its pod. A - * container id is NOT used: a container joined to another's network sees that one's id in - * mountinfo. + * - `container`, the writable layer of the overlay mounted at `/` (its `upperdir`), counted only + * when its path carries a random 64-hex layer id (Docker's `overlay2` driver, containers/storage + * for Podman and CRI-O): a container started again keeps it, across a host reboot as well, and + * no other container ever mounts it, not even one sharing its network (`--network container:`) + * or its pod. A layer named by a sequence number (containerd's overlayfs snapshotter, which + * nodes built alike hand out identically) or a host whose own root is an overlay proves nothing + * beyond one machine, so it is not used; nor is a container id, which a container joined to + * another's network sees in mountinfo. * * A replacement pod, a second container with a fixed `hostname`, or another node matches neither. * Processes that share one network namespace (host networking, pod sidecars, WSL distros, which @@ -86,9 +89,10 @@ function scopeDigest(...parts: string[]): string { } /** - * The `upperdir` of the overlay mounted at `/`, if the root filesystem is one: a container's own - * writable layer. The last root mount wins, as it is the one visible. Any other root filesystem - * (bare host, btrfs/zfs/vfs drivers, fuse-overlayfs) yields nothing. + * The `upperdir` of the overlay mounted at `/` when it names a randomly identified writable layer + * (a `/<64 hex>/` path segment): a container's own layer, unique across machines. The last root + * mount wins, as it is the one visible. Any other root filesystem (bare host, btrfs/zfs/vfs drivers, + * fuse-overlayfs, an overlay host root, containerd's sequence-numbered snapshots) yields nothing. */ export function rootOverlayUpperdirFromMountinfo(mountinfo: string): string | undefined { let upperdir: string | undefined; @@ -99,7 +103,7 @@ export function rootOverlayUpperdirFromMountinfo(mountinfo: string): string | un if (fields[4] !== "/") continue; const [fstype, , superOptions = ""] = line.slice(separator + 3).split(" "); const match = fstype === "overlay" ? /(?:^|,)upperdir=([^,]+)/.exec(superOptions) : null; - upperdir = match?.[1]; + upperdir = match && /\/[0-9a-f]{64}\//.test(match[1]) ? match[1] : undefined; } return upperdir; } diff --git a/structure/runtime.md b/structure/runtime.md index 9ba1096b663..1f938869bcb 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -181,7 +181,7 @@ uninstall still restore. Native client signal cleanup follows the [served-connec `src/cli/tray-proxy.ts` coordinates `ocx restart` and tray restart. A second start requires proof the prior attempt never launched or its child exited, plus confirmed absence under a fresh bounded discovery window; a missing health response alone is insufficient. After an accepted restart, polling continues through uncertain observations until its deadline and requires a new runtime PID on the same port. If the old proxy then disappears, explicit recovery starts it regardless of the Codex autostart preference; a skipped recovery fails rather than reporting success. The Windows tray startup-health probe in `src/tray/windows-tray.ps1` requests termination at its 30-second deadline without waiting on the UI thread. It retains the process through later ticks until actual exit and both redirected readers settle; replacements cannot start while that reference is held. Launch setup failures follow the same cleanup. Tray shutdown alone waits at most 500 ms before disposal. `ocx restart` also restarts a connected client runtime, which serves the same bound `POST /api/system/restart` from its client listener; the CLI sends that request only while the client's 90-second restart budget still fits before replacement observation ends (`CLIENT_RESTART_REQUEST_MIN_REMAINING_MS`), and reports a supervised client's `409 restart_supervised` as "not performed, runtime kept". See [Remote Link](remote-link.md) for the rollback. The Remote Link Child listener checks its startup owner and link against durable client state once per `/readyz`, after readiness probing, and for authenticated machine/link status only on those routes. If they differ, `/readyz` and those status routes answer 503 with `client_state_changed`; the dashboard shows that reason with its recovery guidance. This reports divergence; it does not disconnect the relay or reconstruct a connection. `tests/clients/client-machine-listener.test.ts` covers missing and replaced records. Explicit `ocx disconnect --orphaned-token --yes` in `src/client/orphan-token-recovery.ts` preserves unowned token files only after the start lease, lifecycle lock, liveness proof and config-lock revalidation; Hub, admission-token use and installed or unverified services refuse. Windows closes source handles before rename while private hard links pin identity; a refusal removes a pin only while another name still holds the token, and an extra hard link left by an interrupted run is refused with a message naming the `.identity` pin to remove from the `orphaned-token-*` folder. Missing ownership is never reconstructed. `ocx config import` refuses to replace a connected client record under the config lock. `tests/clients/client-link-connect.test.ts` covers these refusals and local liveness. -The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the PID of the process running the recovery recheck, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name plus either the same network namespace within the same boot (a container restarted inside its Kubernetes pod; namespace inode numbers repeat across boots) or the same writable layer of the overlay mounted at `/` (its `upperdir` in mountinfo: a Docker or Podman container started again, also across a host reboot; no other container mounts it, while a container id is visible to every container joined to that container's network and is therefore not used); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, or another node matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. Processes sharing one network namespace (host networking, pod sidecars, WSL distros in one VM) can match, and there the loopback probe still finds a listening owner; an owner that is alive but no longer listening, an owner exit cleanup that removes records by PID only, and sandboxes that fake namespace numbers or clone a whole VM (gVisor, restored Firecracker/Kata snapshots) are residuals for a shared state volume. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. +The package-tree integrity fence for live package replacement follows the [update transaction contract](ops/docs-and-release.md#package-tree-integrity-fence). After client journal reconciliation, `ocx ensure` in `src/cli/dispatch.ts` treats validated connected-client state as a successful no-op without local startup or a remote health probe; invalid or mismatched state still fails, and disconnected state keeps local ensure. `tests/cli/cli-ensure-client-state.test.ts` pins this dispatch contract so the Codex shim does not warn merely because its proxy is delegated to a hub. Startup journal reconciliation, including replacement after restart/drain, follows the [Codex-home recovery contract](codex-home.md#codex-home): already-original recovery is silent, incomplete recovery retains its journal and warns without claiming restoration, and only complete recovery that rewrites or removes an artifact reports a restore; restart admission, drain deadlines and replacement ownership are unchanged. `src/cli/start-owner-fence.ts` holds the ownership mutation lease through recovery for ordinary and supervised starts. After an early negative probe, `start` and `ensure` recheck with `START_OWNERSHIP_LIVENESS` under that lease through `observeProxyLiveness` in `src/server/proxy-liveness.ts`, probing both available loopback families concurrently for every loopback or wildcard spelling classified after the same startup normalization (including `localhost.` and case variants) within the existing per-family retry ceilings and shared deadline. Numeric loopback classification covers the IPv4 127/8 range, equivalent IPv6 loopback spellings and IPv4-mapped loopback literals; it probes the configured address itself plus the other family, retaining the original bind scope and the responding probe address separately. This recovery classification does not change listener admission or bind policy. `probeEndpointLiveness` and `probePortOwner` retain their existing host selection: other specific loopback literals are checked at the requested address without adopting the opposite family. Only definitive absence permits PID deletion and journal reconciliation: no alive or indeterminate PID from either the PID file or runtime record unless fresh readable command-line evidence confirms it is foreign, no answering runtime owner, and only unanimous aggregate connection refusals or clean foreign identities at probed endpoints. Port zero names no existing listener. Timeouts, resets, unreadable/non-200 responses, PID mismatches and exhausted deadlines remain unknown; startup refuses with a fixed message while preserving PID, journal and shared config. One recovery-only exception covers a specific non-loopback literal IP address (a Tailscale or LAN IP, never a hostname, loopback or wildcard), where a connect to this host's own unbound address can time out instead of being refused: when its probe ends "unknown", `canBindAddress` tries an exclusive listen on that raw socket address (IPv6 unbracketed) and port, closes it at once, and only a successful bind turns that endpoint "dead". The proof shares the discovery deadline: it is skipped once the budget is spent, its timeout is capped by the remaining budget, and a proof finishing past the deadline is discarded. Hostnames remain unknown after an inconclusive probe because name resolution may select different addresses for the probe and bind. Any bind error or timeout leaves the endpoint unknown, so startup still refuses; early (non-observer) discovery never binds. `tests/server/proxy-liveness-nonloopback-bind-proof.test.ts` covers hostnames, bound and unbindable IPs, IPv6 and deadline cases. An orphan runtime PID is checked separately from listener ownership: only a definite missing process or a fresh readable foreign command line permits recovery, while an indeterminate process check, unreadable/empty command line, or OpenCodex invocation remains unknown. One exception covers a record naming the PID of the process running the recovery recheck, which in a container is PID 1 on every start (#8552): it is stale, and skipped as a candidate, only when this process has never written `ocx.pid` or `runtime-port.json` itself (`hasPublishedOwnershipRecords` in `src/config/process-state.ts`) and the runtime record names that PID with an `ownerScope` matching this process's (`runtimeOwnerScope` / `sameRuntimeOwnerScope`), which `writeRuntimePort` stamps only on the owner's own record. On Linux the scope needs the same host name plus either the same network namespace within the same boot (a container restarted inside its Kubernetes pod; namespace inode numbers repeat across boots) or the same randomly named writable layer of the overlay mounted at `/` (an `upperdir` in mountinfo with a 64-hex layer id, as Docker's `overlay2` driver and containers/storage use: the same container started again, also across a host reboot; no other container mounts it. Sequence-numbered containerd snapshots and overlay host roots repeat across machines or sandboxes, and a container id is visible to every container joined to that container's network, so none of them is used); elsewhere the PID space is machine-wide and the host name decides, and unreadable Linux evidence yields a per-process value that matches nothing. A replacement pod, a second container with a fixed hostname, or another node matches neither, so their refusal stands, which matters because such an owner can still be serving the same volume from another network namespace. Processes sharing one network namespace (host networking, pod sidecars, WSL distros in one VM) can match, and there the loopback probe still finds a listening owner; an owner that is alive but no longer listening, an owner exit cleanup that removes records by PID only, sandboxes that fake namespace numbers or clone a whole VM (gVisor, restored Firecracker/Kata snapshots), and non-Linux machines with the same host name syncing one home are residuals for a shared state volume. A record without a scope (an older version), a PID file without a runtime record, and a malformed scope keep the refusal; the scope is not part of record validity, so it can never hide a live owner. `tests/server/proxy-liveness-self-pid-record.test.ts` covers these cases and an `exec` handoff that keeps the PID across `ocx start`. The runtime PID is never supplied to destructive callers merely because it appears in the record; a confirmed foreign PID is never signalled. A discovered live owner preserves both records and its runtime/config bind scope in `LiveProxy.hostname`; omitted or blank hostnames become explicit `127.0.0.1` in both early and recovery discovery, so a later config change cannot replace a default loopback bind. Explicit wildcard scope remains intact, and the concrete responding address is separate `probeHostname` metadata. Integration policy uses the bind scope, so loopback discovery of a wildcard listener does not authorize tokenless Grok registration. Grok start and ensure/restart synchronization also recheck current desired-state admission after catalog discovery, immediately before the synchronous writer in `src/grok/sync.ts`; start additionally rejects work after shutdown begins or cleanup runs. This closes the same-process await gap without turning the desired-state and Grok files into a cross-process transaction. Ensure carries `probeHostname` separately through `src/cli/ensure-desired-integrations.ts` and `src/grok/sync.ts` to `src/grok/inject.ts` for the written destination; an explicit secondary loopback listener keeps its own address and port; after lease release, start applies `decideStartWithLiveOwner` and ensure follows its existing live-owner path. Cross-home sibling marking still precedes journal recovery. The bind/publication fence remains separate, and supervised recovery checks durable service ownership before liveness or mutation. `tests/cli/cli-start-journal-order.test.ts` covers an early miss; `tests/cli/cli-start-owner-absence.test.ts` covers consecutive misses and dead-owner recovery; `tests/server/proxy-liveness-recovery.test.ts` binds classification and existing budgets; `tests/server/proxy-liveness-bind-scope.test.ts` follows a missed early probe through desired-state reconciliation and the real Grok writer for runtime, orphan and configured-port discovery; `tests/server/proxy-liveness-owner-evidence.test.ts` and `tests/cli/cli-start-owner-evidence.test.ts` cover aggregate children, both listener families, shared deadlines and foreign-PID recovery. A busy preferred port is never resolved by starting somewhere else. Both questions a start asks about an existing proxy — the pre-bind owner check and the port-is-busy check in `src/cli/index.ts` diff --git a/tests/server/proxy-liveness-self-pid-record.test.ts b/tests/server/proxy-liveness-self-pid-record.test.ts index 87fb332eba4..cc567b48446 100644 --- a/tests/server/proxy-liveness-self-pid-record.test.ts +++ b/tests/server/proxy-liveness-self-pid-record.test.ts @@ -137,7 +137,7 @@ describe("owner scope comparison", () => { expect(sameRuntimeOwnerScope({ host: "other" }, own)).toBe(false); }); - test("the container key is the root overlay's own writable layer, never a shared id", () => { + test("the container key is the root overlay's own randomly named writable layer, never a shared id", () => { const docker = "/var/lib/docker/overlay2/" + "a".repeat(64) + "/diff"; const root = (fstype: string, options: string) => `1001 1000 0:52 / / rw,relatime master:1 - ${fstype} ${fstype} ${options}`; // A container joined to another's network bind-mounts that one's hostname; it must not count. @@ -147,8 +147,16 @@ describe("owner scope comparison", () => { expect(rootOverlayUpperdirFromMountinfo(root("ext4", "rw,errors=remount-ro") + "\n")).toBeUndefined(); expect(rootOverlayUpperdirFromMountinfo(root("fuse.fuse-overlayfs", "rw,user_id=0") + "\n")).toBeUndefined(); expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l1") + "\n")).toBeUndefined(); + const podman = "/var/lib/containers/storage/overlay/" + "c".repeat(64) + "/diff"; + expect(rootOverlayUpperdirFromMountinfo(root("overlay", `rw,lowerdir=/l,upperdir=${podman},workdir=/w`))).toBe(podman); + // Sequence-numbered containerd snapshots repeat on nodes built alike; an overlay host root is + // shared by every sandbox using it. Neither proves anything beyond one machine. + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/42/fs,workdir=/w"))).toBeUndefined(); + expect(rootOverlayUpperdirFromMountinfo(root("overlay", "rw,lowerdir=/l,upperdir=/media/root-rw/overlay,workdir=/w"))).toBeUndefined(); // The visible root is the last one mounted. - expect(rootOverlayUpperdirFromMountinfo([root("overlay", "rw,upperdir=/old,workdir=/w"), root("overlay", "rw,upperdir=/new,workdir=/w")].join("\n"))).toBe("/new"); + const older = "/var/lib/docker/overlay2/" + "d".repeat(64) + "/diff"; + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `rw,upperdir=${older},workdir=/w`), root("overlay", `rw,upperdir=${docker},workdir=/w`)].join("\n"))).toBe(docker); + expect(rootOverlayUpperdirFromMountinfo([root("overlay", `rw,upperdir=${older},workdir=/w`), root("overlay", "rw,upperdir=/var/lib/containerd/snapshots/7/fs,workdir=/w")].join("\n"))).toBeUndefined(); }); });