Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion scripts/test-layout/layout-start-ownership.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,7 @@
"cli-start-owner-absence.test.ts": "cli",
"cli-start-owner-evidence.test.ts": "cli",
"proxy-liveness-owner-evidence.test.ts": "server",
"proxy-liveness-recovery.test.ts": "server"
"proxy-liveness-recovery.test.ts": "server",
"proxy-liveness-bind-scope.test.ts": "server",
"system-restart-ownership.test.ts": "server"
}
16 changes: 9 additions & 7 deletions src/cli/ensure-desired-integrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export interface EnsureDesiredIntegrationsDeps {
syncGrokConfig: (
port: number,
config: OcxConfig,
opts?: { hostname?: string },
opts?: { hostname?: string; probeHostname?: string },
) => Promise<GrokInjectResult>;
removeDesktop3pStandardPivot: typeof removeDesktop3pStandardPivot;
removeDesktopFirstParty?: typeof removeDesktopFirstParty;
Expand All @@ -67,7 +67,7 @@ export interface EnsureDesiredIntegrationsDeps {
async function defaultSyncGrokConfig(
port: number,
config: OcxConfig,
opts: { hostname?: string } = {},
opts: { hostname?: string; probeHostname?: string } = {},
): Promise<GrokInjectResult> {
const { syncGrokConfig } = await import("../grok/sync");
return syncGrokConfig(port, config, opts);
Expand Down Expand Up @@ -100,7 +100,7 @@ function io(deps: EnsureDesiredIntegrationsDeps): {
*/
export async function ensureGrokFenceMatchesDesired(
port: number,
opts: { hostname?: string } = {},
opts: { hostname?: string; probeHostname?: string } = {},
deps: EnsureDesiredIntegrationsDeps = productionDeps,
): Promise<void> {
const config = deps.loadConfig();
Expand Down Expand Up @@ -132,7 +132,8 @@ export async function ensureGrokFenceMatchesDesired(
const g = await deps.syncGrokConfig(
port,
config,
hostname !== undefined ? { hostname } : {},
{ ...(hostname !== undefined ? { hostname } : {}),
...(opts.probeHostname !== undefined ? { probeHostname: opts.probeHostname } : {}) },
);
if (g.changed) log(" + Grok Build config updated (~/.grok/config.toml)");
else if (!g.ok) error(`⚠️ ${g.message}`);
Expand Down Expand Up @@ -227,12 +228,12 @@ export async function ensureClaudeDesktopMatchesDesired(
}

export type EnsureDesiredIntegrationsBranch =
| { kind: "live"; hostname?: string }
| { kind: "live"; hostname?: string; probeHostname?: string }
| { kind: "spawned" };

/**
* Reconcile the two external integration files after either ensure race window.
* Only the live proxy's observed bind host crosses this boundary; persisted
* The live proxy's bind scope and responding address cross this boundary; persisted
* config is deliberately loaded inside each mutation helper.
*/
export async function reconcileEnsureDesiredIntegrations(
Expand All @@ -243,7 +244,8 @@ export async function reconcileEnsureDesiredIntegrations(
const liveHost = branch.kind === "live" ? branch.hostname : undefined;
await ensureGrokFenceMatchesDesired(
port,
liveHost ? { hostname: liveHost } : {},
{ ...(liveHost ? { hostname: liveHost } : {}),
...(branch.kind === "live" && branch.probeHostname !== undefined ? { probeHostname: branch.probeHostname } : {}) },
deps,
);
await ensureClaudeDesktopMatchesDesired(deps);
Expand Down
10 changes: 5 additions & 5 deletions src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -854,13 +854,13 @@ async function handleEnsure(options: { existingIsSuccess?: boolean; forceStart?:
const systemEnv = await injectSystemEnv(live.port, config).catch(() => ({ injected: false }));
if (siblingOfLivePort() === null) reportShellHookFailure(reconcileShellHook(systemEnv.injected));
if (!systemEnv.injected) await syncClaudeAgentDefsAtProxyStartup(config, live.port);
// Refresh the Grok Build fence too (same contract as start). live.hostname is the
// hostname the running proxy actually bound — config.hostname may have drifted.
// The reconciler re-reads immediately before each client-file mutation; only
// the live proxy's observed bind host is safe to carry across this boundary.
// Refresh the Grok Build fence too (same contract as start). Keep the observed
// bind scope for admission and its responding address for the destination.
// The reconciler re-reads immediately before each client-file mutation;
// config.hostname may have drifted from the running proxy's bind scope.
if (siblingOfLivePort() === null) await reconcileEnsureDesiredIntegrations(
live.port,
{ kind: "live", hostname: live.hostname },
{ kind: "live", hostname: live.hostname, probeHostname: live.probeHostname },
);
console.log(`✅ Proxy running on port ${live.port}`);
return true;
Expand Down
3 changes: 2 additions & 1 deletion src/grok/inject.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1119,6 +1119,7 @@ export function injectGrokConfig(
opts: {
grokHome?: string;
hostname?: string;
probeHostname?: string;
refreshOnly?: { admit: () => boolean };
excluded?: ReadonlySet<string>;
/** Unfiltered known ids used only to distinguish hidden current models from retired ones. */
Expand Down Expand Up @@ -1215,7 +1216,7 @@ export function injectGrokConfig(
const region = orphans.length > 0 ? findManagedRegion(content) : originalRegion;

const buildCandidate = (reservedAliases: ReadonlySet<string>): string => {
const block = buildGrokManagedBlock(port, models, opts.hostname, reservedAliases, opts.excluded);
const block = buildGrokManagedBlock(port, models, opts.probeHostname ?? opts.hostname, reservedAliases, opts.excluded);
let candidate: string;
if (region) {
candidate = content.slice(0, region.start) + block + content.slice(region.end);
Expand Down
7 changes: 5 additions & 2 deletions src/grok/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ async function defaultFetchAllModels(config: OcxConfig): Promise<CatalogModel[]>
* Build the model list and inject the fenced block. `hostname` should be the hostname the
* RUNNING proxy actually bound (live.hostname from proxy-liveness for ensure's live branch;
* config.hostname for a freshly spawned start) — a stale config.hostname could otherwise
* name a host the process never bound.
* name a host the process never bound. `probeHostname` is only the reachable destination;
* it never replaces bind scope for admission policy or an explicit secondary listener.
*/
export async function syncGrokConfig(
port: number,
config: OcxConfig,
opts: { hostname?: string; grokHome?: string; refreshOnly?: { admit: () => boolean } } = {},
opts: { hostname?: string; probeHostname?: string; grokHome?: string; refreshOnly?: { admit: () => boolean } } = {},
deps: GrokSyncDeps = { fetchAllModels: defaultFetchAllModels, injectGrokConfig },
): Promise<GrokInjectResult> {
let projection: ReturnType<typeof projectGrokCatalog>;
Expand All @@ -57,6 +58,8 @@ export async function syncGrokConfig(
hostname: target.requiresAdmissionToken
? (opts.hostname ?? config.hostname)
: targetUrl.hostname,
...(opts.probeHostname !== undefined && !config.unauthenticatedLoopbackListener?.enabled
? { probeHostname: opts.probeHostname } : {}),
...(opts.grokHome !== undefined ? { grokHome: opts.grokHome } : {}),
...opts.refreshOnly ? { refreshOnly: opts.refreshOnly } : {},
excluded: new Set(config.grokExcludedModels ?? []),
Expand Down
4 changes: 4 additions & 0 deletions src/server/management/system-restart.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import {
isDesktopSupervised,
} from "../../lib/system-restart-contract";
import { spawnReplacementStart } from "../restart-replacement";
import { retireRestartParentOwnership } from "../restart-parent-ownership";

export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract";
export { waitForReplacementReady, type ReplacementReadinessIo } from "../restart-replacement";
Expand Down Expand Up @@ -289,6 +290,9 @@ function spawnDetachedStart(
let env: NodeJS.ProcessEnv;
try {
env = replacementStartEnvironment(waitForHealthBeforeParentExit);
// Prepare the sibling handoff while its runtime record still exists. Only completed
// drain + listener stop may release parent ownership before waiting for child health.
if (waitForHealthBeforeParentExit) retireRestartParentOwnership(port);
} catch (err) {
return Promise.reject(err);
}
Expand Down
16 changes: 12 additions & 4 deletions src/server/proxy-liveness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,10 @@ type LivenessFetch = (
export interface LiveProxy {
pid: number | null;
port: number;
/** Raw bind hostname the probe succeeded against; compose URLs via `probeHostname`. */
/** Candidate bind scope, with the default loopback made explicit; never a probe alias. */
hostname?: string;
/** Successful recovery probe address, when discovery checked both loopback families. */
probeHostname?: string;
/** Whether the successful probe used runtime-port metadata or the configured listen port. */
source: "runtime" | "config";
/**
Expand Down Expand Up @@ -202,7 +204,7 @@ export type HomeOwnershipProof = "proven" | "refuted" | "indeterminate";
export async function proveLiveProxyOwnedByHome(live: LiveProxy, io: LivenessIo = {}): Promise<HomeOwnershipProof> {
if (live.pid === null) return "indeterminate";
return attestFencedIdentity(
`http://${probeHostname(live.hostname)}:${live.port}/healthz`,
`http://${probeHostname(live.probeHostname ?? live.hostname)}:${live.port}/healthz`,
live.port,
live.pid,
io,
Expand Down Expand Up @@ -519,18 +521,21 @@ async function discoverLiveProxy(io: LivenessIo, observer?: {
deadlineAt !== undefined && nowFn() >= deadlineAt;
const expired = (): null => { observer?.endpoint("unknown"); return null; };
const probe = async (port: number, opts: { hostname?: string; expectedPid?: number }) => {
// Missing/blank means an actual loopback bind, not a later config's bind scope.
const bindHostname = opts.hostname?.trim() || "127.0.0.1";
const hostname = normalizeBindHostname(opts.hostname);
if (!observer || (!isLoopbackHostname(hostname) && !isWildcardHostname(hostname))) {
const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer?.endpoint);
return identity ? { ...identity, hostname: opts.hostname } : null;
return identity ? { ...identity, hostname: bindHostname,
...(observer ? { probeHostname: probeHostname(opts.hostname) } : {}) } : null;
}
// Parallel families share the existing per-probe attempts/timeout and absolute
// deadline. A slow family cannot multiply the serial discovery budget.
const hosts = loopbackProbeHosts(hostname).filter(host =>
(io.loopbackAvailabilityFn ?? loopbackFamilyAvailability)(host) !== "unavailable");
const identities = await Promise.all(hosts.map(async hostname => {
const identity = await proxyIdentityAtObserved(port, { ...opts, hostname }, probeIo, observer.endpoint);
return identity ? { ...identity, hostname } : null;
return identity ? { ...identity, hostname: bindHostname, probeHostname: hostname } : null;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}));
return identities.find(identity => identity !== null) ?? null;
};
Expand Down Expand Up @@ -568,6 +573,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: {
pid: trusted,
port: runtime.port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "runtime",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
Expand All @@ -593,6 +599,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: {
pid: verifiedReportedPid(identity.pid),
port: record.port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "runtime",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
Expand All @@ -612,6 +619,7 @@ async function discoverLiveProxy(io: LivenessIo, observer?: {
pid: verifiedReportedPid(identity.pid) ?? killablePid(pid),
port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "config",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
Expand Down
49 changes: 49 additions & 0 deletions src/server/restart-parent-ownership.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { lstatSync } from "node:fs";
import {
getPidPath, getRuntimePortPath, readPidFileValue, readRuntimePort, removePid, removeRuntimePort,
} from "../config/process-state";
import { acquireOwnershipMutationLease } from "../service/ownership-mutation-lease.mjs";
import { serviceStatePaths } from "../service/state";

function refusal(): Error {
return Object.assign(new Error("Restart parent ownership could not be retired safely"), {
code: "RESTART_PARENT_OWNERSHIP_UNVERIFIED",
});
}

/** Only ENOENT proves absence; an unreadable, malformed or linked record is not ours to retire. */
function recordExists(path: string): boolean {
try {
if (!lstatSync(path).isFile()) throw refusal();
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
throw refusal();
}
}

/**
* Called only after completed drain and successful listener stop, before waiting for child health.
* The parent is still alive, but no longer owns a listener or startup recovery. Retire only this
* process's records under the same lease that serializes child recovery and publication. Ordinary
* live/unknown owners retain the startup fence; no child marker grants an exemption from it.
*/
export function retireRestartParentOwnership(port?: number): void {
const lease = acquireOwnershipMutationLease(serviceStatePaths());
try {
const pidPath = getPidPath(), runtimePath = getRuntimePortPath();
const hasPid = recordExists(pidPath), hasRuntime = recordExists(runtimePath);
const runtime = readRuntimePort();
if ((hasPid && readPidFileValue() !== process.pid)
|| (hasRuntime && (!runtime || runtime.pid !== process.pid
|| (port !== undefined && runtime.port !== port)))) throw refusal();
// Keep the alive-PID fence if runtime retirement fails. The helpers also recheck the PID
// immediately before unlink, so a successor's record is never unconditional cleanup.
if (hasRuntime) removeRuntimePort(process.pid);
if (recordExists(runtimePath)) throw refusal();
if (hasPid) removePid(process.pid);
if (recordExists(pidPath)) throw refusal();
} finally {
lease.release();
}
}
52 changes: 26 additions & 26 deletions structure/ops/service-and-sidecars.md
Original file line number Diff line number Diff line change
Expand Up @@ -540,32 +540,32 @@ API opt-in; dashboard, CLI, tray, join and automatic restarts keep their default
turn may have an unknown upstream outcome and is not automatically replayed.
`tests/server/system-restart-admission.test.ts` covers this contract and active-turn preservation.

A dashboard drain-and-restart (`src/server/management/system-restart.ts`, which is also the restart
after a join into a Child) and the client runtime's standalone recycle (`src/client/runtime.ts`)
replace their process through `src/server/restart-replacement.ts`. Every replacement `ocx start`
carries `OCX_RESTART_PARENT_PID`. `handleStart` consumes the marker before its first probe and
honors it only when it names the process's real parent. When the live owner that probe finds is
exactly that pid, `decideStartWithLiveOwner` answers `await-parent`, and `src/cli/restart-handoff.ts`
waits up to 30 seconds for the parent to exit or stop answering (re-probing once a second) before it
probes again; a parent that outlives the wait is refused like any live proxy. An ordinary start
carries no marker and probes once.

Only a handoff that waits for health (the drain completed) respawns a replacement that exits before
it answers, at most twice, inside the one 70-second readiness budget. A spawn error is not retried.
A parent-exit handoff (drain deadline, failed or rejected drain, listener-stop fallback) resolves as
soon as the child spawned: the parent must exit to release what the replacement waits for, so it
cannot watch for an early exit. The replacement's `await-parent` wait and the link-mode port reclaim
cover the known transient causes there. Any other early exit on that path is a residual: no proxy
serves the port until something runs `ocx start` again, and Codex keeps pointing at that port.

The replacement's stdout and stderr go to `<configDir>/restart-handoff.log`: mode 0600, opened
without following a symlink, recorded as an owned config path, and bounded at 256 KiB on both sides.
A handoff empties the file before it opens it. The replacement keeps writing to it for its whole
life, so the parent hands it `OCX_RESTART_HANDOFF_LOG=1` along with the file; `handleStart` consumes
the flag and arms one unref'd 60-second timer (`armRestartHandoffLogCap`) that lstats the file and
empties it at the cap through a fresh descriptor. A start without the flag arms nothing. The parent
writes only timestamps, pids, ports, attempt counts, exit codes and errno labels, never an
environment value.
A dashboard drain-and-restart (`src/server/management/system-restart.ts`, including a join into a Child) and the
client runtime's standalone recycle (`src/client/runtime.ts`) use `src/server/restart-replacement.ts`. Every child
carries `OCX_RESTART_PARENT_PID`; `handleStart` consumes it before probing and honors only its real parent PID.
If that parent still answers, `decideStartWithLiveOwner` returns `await-parent`: `src/cli/restart-handoff.ts` waits
up to 30 seconds, re-probing each second, then refuses a surviving owner. An ordinary start probes once without a marker.

After completed drain and successful listener stop, the unsupervised dashboard parent prepares its sibling handoff,
then `src/server/restart-parent-ownership.ts` retires only its own PID/runtime records under the ownership mutation
lease before spawning and waiting for child health. Foreign PIDs, mismatched runtime ports, malformed, linked or
unreadable records and incomplete removal refuse the handoff. Only ENOENT proves a missing record. The lease is
released before spawning, so child recovery and publication can acquire it. The normal fence still refuses live and
unknown owners; no child marker bypasses it. Exact-PID exit cleanup preserves the successor's records.
`tests/server/system-restart-ownership.test.ts` covers refusal controls and real processes, drain, alive-parent endpoint
refusal, replacement health and parent exit; its fixture does not start providers or exercise dashboard HTTP authentication.

A completed-drain health wait retries a replacement that exits before answering at most twice within one 70-second
readiness budget; spawn errors are not retried. A parent-exit handoff (drain deadline, failed or rejected drain,
listener-stop fallback) resolves as soon as the child spawns, because the parent must exit to release what the child
waits for. It cannot watch for early exit. The child's `await-parent` wait and link-mode port reclaim cover the known
transient causes. Other early exits leave the port unserved until another `ocx start`, while Codex keeps pointing there.

Replacement output goes to `<configDir>/restart-handoff.log`: private mode 0600, opened without following symlinks,
recorded as an owned path and bounded at 256 KiB. Handoff empties an already-full file. `OCX_RESTART_HANDOFF_LOG=1`
tells `handleStart` to arm `armRestartHandoffLogCap`, an unref'd 60-second timer which lstats and empties a full log
through a fresh descriptor. An ordinary start arms no timer. Parent log lines carry only timestamps, PIDs, ports,
attempt counts, exit codes and errno labels, never environment values.

When every attempt fails while client state is `connected` (a join has committed), the parent marks
recycling before `exit(1)`, so its exit cleanup keeps the Codex routing `connectClient` wrote
Expand Down
Loading
Loading