fix(start): recover a same-host record left by an earlier lifetime of this PID

docs-site/src/content/docs/guides/remote-hub.md

@@ -745,6 +745,27 @@ both named volumes and a synthetic catalog survive replacement. This check does
 real provider account, OAuth callback, custom mount migration, or every CPU architecture; perform
 the authenticated routed-response check above for your deployment.
 
+### Restarts after an unclean exit
+
+A container runs the proxy as PID 1 on every start. After an unclean exit (an OOM kill, a crash, a
+node failure) the state volume can still hold `ocx.pid` and `runtime-port.json` naming PID 1, and
+the next start has to decide whether that owner is gone. It recovers on its own when the record was
+written by the same container in an earlier run, by an OpenCodex version that records where its
+owner runs: a Kubernetes container restarted inside its pod while the node stays up, or the same
+Docker or Podman container started again (`docker restart`, a restart policy, including after a
+host reboot) when its writable layer has a random id, as with Docker's `overlay2` storage driver
+and Podman. Docker's containerd image store numbers its layers instead, so a restart there refuses.
+
+Every other case refuses with `Proxy ownership is unverified; startup refused`, because the old
+owner could still be serving the same volume from another container: a replacement Kubernetes pod,
+a recreated Compose container, a pod that comes back after a node reboot, another node, or a record
+written by an older version (the first restart after an upgrade). Confirm that nothing else serves this state directory, then move
+`runtime-port.json` and `ocx.pid` aside once and start again. Run a single writer per state
+volume: a Kubernetes `Deployment` that shares a volume should use `strategy: Recreate` (or
+`maxSurge: 0`) so the old pod stops before the new one starts. Do not share one state volume between
+processes in one network namespace (host networking, sidecars, several WSL distros), between
+sandboxes restored from one VM snapshot, or between machines with the same host name.
+
 ## Rollback
 
 Inspect existing Serve mappings before changing them. `tailscale serve reset` removes every mapping

scripts/test-layout/layout-start-ownership.json

@@ -4,6 +4,7 @@
   "proxy-liveness-owner-evidence.test.ts": "server",
   "proxy-liveness-recovery.test.ts": "server",
   "proxy-liveness-bind-scope.test.ts": "server",
+  "proxy-liveness-self-pid-record.test.ts": "server",
   "proxy-liveness-nonloopback-bind-proof.test.ts": "server",
   "system-restart-ownership.test.ts": "server"
 }

src/config/process-state.ts

@@ -1,5 +1,7 @@
 import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readlinkSync, unlinkSync } from "node:fs";
+import { hostname } from "node:os";
 import { join } from "node:path";
 import { isLocalAttestationSecret } from "../lib/local-management-attestation";
 import { assertNotRealHomeUnderTest } from "../lib/test-home-guard";
@@ -27,8 +29,122 @@ function ensureProcessStateDir(): void {
   else hardenConfigDir();
 }
 
+/**
+ * Whether this process has written `ocx.pid` or `runtime-port.json` in its own lifetime (#8552).
+ *
+ * Set before the write, deliberately even when the write then throws, and never cleared: a record
+ * this process wrote (or may have written) and later removed still means its PID once named a live
+ * owner of this home, so it can never be told apart from an earlier lifetime of the same PID number.
+ * Moving the assignment after the write would let a failed write fail open.
+ */
+let publishedOwnershipRecords = false;
+
+export function hasPublishedOwnershipRecords(): boolean {
+  return publishedOwnershipRecords;
+}
+
+/**
+ * Where the owner of a runtime record ran, as digests that store none of their inputs (#8552).
+ *
+ * Every container runs the proxy as PID 1, so a PID alone cannot say whether a record came from
+ * this container's previous run or from another container serving the same state volume. The
+ * `host` name must match, and on Linux at least one of two stronger facts as well
+ * ({@link sameRuntimeOwnerScope}):
+ *
+ * - `network` with `boot`, the network namespace within one boot: a container restarted inside
+ *   its Kubernetes pod keeps it, because the pod sandbox owns it. Namespace inode numbers repeat
+ *   across machines and boots, so the boot id has to match too;
+ * - `container`, the writable layer of the overlay mounted at `/` (its `upperdir`), counted only
+ *   when its path carries a random 64-hex layer id (Docker's `overlay2` driver, containers/storage
+ *   for Podman and CRI-O): a container started again keeps it, across a host reboot as well, and
+ *   no other container ever mounts it, not even one sharing its network (`--network container:`)
+ *   or its pod. A layer named by a sequence number (containerd's overlayfs snapshotter, which
+ *   nodes built alike hand out identically) or a host whose own root is an overlay proves nothing
+ *   beyond one machine, so it is not used; nor is a container id, which a container joined to
+ *   another's network sees in mountinfo.
+ *
+ * A replacement pod, a second container with a fixed `hostname`, or another node matches neither.
+ * Processes that share one network namespace (host networking, pod sidecars, WSL distros, which
+ * share one VM) can match on `network`; for them the loopback probe still finds a listening owner.
+ * Elsewhere the PID space is the whole machine and the host name suffices. Linux evidence that
+ * cannot be read yields a per-process value that matches no other lifetime, so the self-record
+ * proof fails closed.
+ */
+export interface RuntimeOwnerScope {
+  host: string;
+  boot?: string;
+  network?: string;
+  container?: string;
+}
+
+let cachedRuntimeOwnerScope: RuntimeOwnerScope | undefined;
+
+export function runtimeOwnerScope(): RuntimeOwnerScope {
+  cachedRuntimeOwnerScope ??= computeRuntimeOwnerScope();
+  return cachedRuntimeOwnerScope;
+}
+
+function scopeDigest(...parts: string[]): string {
+  return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 32);
+}
+
+/**
+ * The `upperdir` of the overlay mounted at `/` when it names a randomly identified writable layer
+ * (a `/<64 hex>/` path segment): a container's own layer, unique across machines. The last root
+ * mount wins, as it is the one visible. Any other root filesystem (bare host, btrfs/zfs/vfs drivers,
+ * fuse-overlayfs, an overlay host root, containerd's sequence-numbered snapshots) yields nothing.
+ */
+export function rootOverlayUpperdirFromMountinfo(mountinfo: string): string | undefined {
+  let upperdir: string | undefined;
+  for (const line of mountinfo.split("\n")) {
+    const separator = line.indexOf(" - ");
+    if (separator < 0) continue;
+    const fields = line.slice(0, separator).split(" ");
+    if (fields[4] !== "/") continue;
+    const [fstype, , superOptions = ""] = line.slice(separator + 3).split(" ");
+    const match = fstype === "overlay" ? /(?:^|,)upperdir=([^,]+)/.exec(superOptions) : null;
+    upperdir = match && /\/[0-9a-f]{64}\//.test(match[1]) ? match[1] : undefined;
+  }
+  return upperdir;
+}
+
+function readOptional(read: () => string): string | undefined {
+  try { return read(); } catch { return undefined; }
+}
+
+function computeRuntimeOwnerScope(): RuntimeOwnerScope {
+  if (process.platform !== "linux") return { host: scopeDigest(process.platform, hostname()) };
+  const boot = readOptional(() => readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim());
+  const network = readOptional(() => readlinkSync("/proc/self/ns/net"));
+  const container = readOptional(() => rootOverlayUpperdirFromMountinfo(readFileSync("/proc/self/mountinfo", "utf8")) ?? "");
+  const scope: RuntimeOwnerScope = { host: scopeDigest("linux", hostname()) };
+  if (boot && /^[0-9a-f-]{36}$/.test(boot) && network && /^net:\[\d+\]$/.test(network)) {
+    scope.boot = scopeDigest("boot", boot);
+    scope.network = scopeDigest("net", network);
+  }
+  if (container) scope.container = scopeDigest("container", container);
+  if (!scope.network && !scope.container) return { host: scopeDigest("unverifiable", randomUUID()) };
+  return scope;
+}
+
+/**
+ * Whether a recorded owner scope proves the record was written where this process runs. Only an
+ * exact, well-formed match counts; anything else (absent, malformed, partial) is "no".
+ */
+export function sameRuntimeOwnerScope(recorded: unknown, own: RuntimeOwnerScope): boolean {
+  if (!recorded || typeof recorded !== "object" || Array.isArray(recorded)) return false;
+  const record = recorded as Record<string, unknown>;
+  if (typeof record.host !== "string" || record.host !== own.host) return false;
+  if (own.network === undefined && own.container === undefined) {
+    return record.boot === undefined && record.network === undefined && record.container === undefined;
+  }
+  return (own.network !== undefined && record.network === own.network && own.boot !== undefined && record.boot === own.boot)
+    || (own.container !== undefined && record.container === own.container);
+}
+
 export function writePid(pid: number): void {
   ensureProcessStateDir();
+  publishedOwnershipRecords = true;
   atomicWriteFile(getPidPath(), String(pid));
 }
 
@@ -45,9 +161,16 @@ export type RuntimePortState = {
   /**
    * The live proxy's port when this runtime is a sibling instance started beside it
    * (`src/codex/sibling-start.ts`). `ocx stop` reads it to leave shared client routing alone.
-   * Absent for every other runtime, so those records keep their bytes.
+   * Absent for every other runtime.
    */
   siblingOfPort?: number;
+  /**
+   * {@link runtimeOwnerScope} of the owner, stamped by {@link writeRuntimePort} when this process
+   * writes its own record. Not validated here: a malformed value must not hide a live owner from
+   * status, stop or liveness, and the one reader that uses it (`observeProxyLiveness`) accepts
+   * only an exact match ({@link sameRuntimeOwnerScope}).
+   */
+  ownerScope?: RuntimeOwnerScope;
 };
 
 export function isValidRuntimePortState(value: unknown): value is RuntimePortState {
@@ -70,7 +193,12 @@ export function isValidRuntimePortState(value: unknown): value is RuntimePortSta
 
 export function writeRuntimePort(state: RuntimePortState): void {
   ensureProcessStateDir();
-  atomicWriteFile(getRuntimePortPath(), JSON.stringify(state, null, 2) + "\n");
+  publishedOwnershipRecords = true;
+  // Only the owner can say where it runs; a record written on another process's behalf keeps
+  // whatever identity it already carried.
+  const ownerScope = state.pid === process.pid ? runtimeOwnerScope() : state.ownerScope;
+  const record = { ...state, ...(ownerScope === undefined ? {} : { ownerScope }) };
+  atomicWriteFile(getRuntimePortPath(), JSON.stringify(record, null, 2) + "\n");
   // The record proves this home's owner only to a reader that knows where it lives.
   // One pointer in the shared registry makes the record findable from every home;
   // it is best-effort because ownership never depends on the registry write landing.

src/server/proxy-liveness.ts

@@ -12,7 +12,10 @@
 import { loadConfig } from "../config";
 import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target";
 import { normalizeBindHostname } from "./bind-host";
-import { isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort, verifyPidIdentity } from "../config/process-state";
+import {
+  hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort,
+  runtimeOwnerScope, sameRuntimeOwnerScope, verifyPidIdentity, type RuntimeOwnerScope,
+} from "../config/process-state";
 import { createServer, isIP } from "node:net";
 import { loopbackFamilyAvailability } from "../lib/loopback-family";
 import {
@@ -93,7 +96,9 @@ export interface LivenessIo {
    * socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget.
    */
   bindProofFn?: (hostname: string, port: number, timeoutMs: number) => Promise<boolean>;
-  readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string } | null;
+  readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; ownerScope?: unknown } | null;
+  /** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */
+  ownProcessFn?: () => OwnProcess;
   configFn?: () => { port?: number; hostname?: string };
   timeoutMs?: number;
   /**
@@ -513,21 +518,43 @@ export type ProxyLivenessObservation =
   | { kind: "live"; live: LiveProxy }
   | { kind: "absent" | "unknown"; live: null };
 
+export interface OwnProcess {
+  pid: number;
+  /** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */
+  publishedOwnership: boolean;
+  scope: RuntimeOwnerScope;
+}
+
+function currentOwnProcess(): OwnProcess {
+  return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), scope: runtimeOwnerScope() };
+}
+
 /** Recovery additionally covers both available loopback families and fresh PID identity. */
 export async function observeProxyLiveness(io: LivenessIo = {}): Promise<ProxyLivenessObservation> {
-  let alivePid: number | null = null, runtimePid: number | undefined, unknown = false;
+  let alivePid: number | null = null, runtimePid: number | undefined, runtimeScope: unknown, unknown = false;
   const live = await discoverLiveProxy(io, {
     pid: pid => { alivePid = pid; },
-    runtimePid: pid => { runtimePid = pid; },
+    runtimePid: (pid, ownerScope) => { runtimePid = pid; runtimeScope = ownerScope; },
     endpoint: result => { if (result === "unknown") unknown = true; },
   });
   if (live) return { kind: "live", live };
+  // #8552: in a container the proxy is PID 1 on every start, so the record the previous container
+  // left names this very process; signalling it succeeds and its command line is ocx. Such a
+  // record is stale only when this process never published ownership itself (so the writer was an
+  // earlier lifetime of the PID number) AND the runtime record's owner ran where this process runs
+  // (same host and boot, plus the same network namespace or container). That scope check keeps a
+  // RollingUpdate safe: the old pod is also PID 1 and its listener sits in another network
+  // namespace, which the endpoint probe cannot see. A record without a scope (an older version,
+  // which may still be serving during an upgrade) keeps the refusal.
+  const own = (io.ownProcessFn ?? currentOwnProcess)();
+  const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && sameRuntimeOwnerScope(runtimeScope, own.scope);
   const candidates = new Set<number>();
   if (alivePid !== null) candidates.add(alivePid);
   if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid);
   for (const pid of candidates) {
     if (unknown) break;
     if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; }
+    if (ownEarlierLifetime && pid === own.pid) continue;
     if (pid !== alivePid) {
       try { process.kill(pid, 0); } catch (error) {
         if ((error as NodeJS.ErrnoException).code === "ESRCH") continue;
@@ -557,7 +584,7 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
 
 async function discoverLiveProxy(io: LivenessIo, observer?: {
   pid: (pid: number | null) => void;
-  runtimePid: (pid: number | undefined) => void;
+  runtimePid: (pid: number | undefined, ownerScope: unknown) => void;
   endpoint: (result: EndpointLiveness) => void;
 }): Promise<LiveProxy | null> {
   // Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too
@@ -659,7 +686,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: {
   // sibling command, manual deletion). The runtime record still says where it listens —
   // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it.
   const record = readRuntimeFn();
-  observer?.runtimePid(record?.pid);
+  observer?.runtimePid(record?.pid, record?.ownerScope);
   if (record?.port && record.port !== probedPort) {
     if (budgetExhausted()) return expired();
     const expectedPid = typeof record.pid === "number" ? record.pid : undefined;