diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index 6cdb10f4d2f..0792854aa24 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -114,7 +114,7 @@ is visible) and never forces an upstream refresh. | **Native reasoning retention** | Independently opt into reasoning ciphertext forwarding on native model or account changes. Both default off; account retention is experimental. See [the configuration scope](/reference/configuration/server/#native-reasoning-retention). | | **Sub-agent delegation** | Choose a native or routed model and optional reasoning effort shared by OpenCodex delegation guidance and the separate native-default opt-in. This is not a proxy-side per-spawn router; see below. | | **Sidecars** | Choose the web-search model and effort plus the vision-description model. Changes apply on the next request. | -| **Maintenance** | Resync the Codex model catalog, inspect project-local config bypass warnings, check the latest or preview release, and run an update with optional proxy restart. In the desktop shell, its update entry opens the native app update page instead of running the package updater. | +| **Maintenance** | Resync the Codex model catalog, inspect project-local config bypass warnings, check the latest or preview release, and run an update with optional proxy restart. With restart on, Codex routing and its model catalog stay in place while the package is replaced, so a Codex app that loads them during the update still sees your routed models; the restarted proxy takes them over, and `ocx stop` restores native Codex if it never comes back. In the desktop shell, its update entry opens the native app update page instead of running the package updater. | | **Startup safety** | Show whether injected Codex routing survives a restart, with separate service and launcher-shim health plus exact repair commands. | | **Windows tray** | Install a per-user login tray for one-click proxy start, stop, restart, dashboard access, and status. The tray is a controller, not a proxy restart service. | | **Codex autostart** | Allow an already-installed Codex launcher shim to run `ocx ensure`. This toggle does not install a shim or background service. | diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index 9f80c44487b..cabb3012544 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -124,8 +124,11 @@ failure to restore shared client configuration keeps the stop failed and its rec carries the outcome class (`stopped`, `not-running`, `history-incomplete`, `history-deferred`, or `failed`), the service and proxy classifications, whether the runtime is down, and a stable one-line message. Exit codes are identical with and without `--json`: 0 on success, 1 on failure, -79 when only Codex history cleanup did not complete, and 80 when the shared teardown was deferred -and is still owed. +79 when only Codex history cleanup did not complete, 80 when the shared teardown was deferred +and is still owed, and 81 when a dashboard update that will restart the proxy handed Codex routing +and its catalog to the restarted proxy instead of restoring native Codex. With 81 the proxy is +down, and the restarted proxy releases the kept teardown after its first sync; if it never starts, +the next `ocx stop` restores native Codex. For a desktop-approved takeover, `ocx stop --json` also accepts the complete set of `--expect-*` values supplied by `ocx resolve --json`. On Windows, this guarded stop can diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index bc228afaffb..8d1790a74bf 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1848,7 +1848,7 @@ "update-refresh.test.ts": "update", "update-restart-lease.test.ts": "update", "update-stop-classification.test.ts": "update", - "update-stop-first.test.ts": "update", + "update-stop-first.test.ts": "update", "update-teardown-handover.test.ts": "update", "update-transactional-leftovers.test.ts": "update", "update-transactional.test.ts": "update", "update-tray-handoff.test.ts": "update", diff --git a/src/cli/cross-home-owner.ts b/src/cli/cross-home-owner.ts index a296d7698cb..1d70a35fdf7 100644 --- a/src/cli/cross-home-owner.ts +++ b/src/cli/cross-home-owner.ts @@ -9,6 +9,9 @@ import { detectCodexRoutingDrift } from "../codex/routing-drift"; import { readBoundedCodexConfig } from "../codex/inject/bounded-config-reader"; import { currentExternalCodexModelProvider } from "../codex/inject"; import { reconcileJournal } from "../codex/journal"; +import { shouldSyncCodexOnStart } from "../codex/desired-state"; +import { loadConfig } from "../config"; +import { updateTeardownHandoverPending } from "../update/teardown-handover"; import { markSiblingStart, siblingOfLivePort } from "../codex/sibling-start"; import { readClientConnectionState } from "../client/state"; import { findManagedRegion, resolveGrokHome } from "../grok/inject"; @@ -350,5 +353,14 @@ export function reconcileStartupJournal(): void { const clientState = readClientConnectionState(); reconcileJournal(clientState.kind === "connected" ? { activeClientApiKeyId: clientState.value.apiKeyId } - : undefined); + : { adoptUpdateHandover: updateTeardownHandoverPending() && startupSyncEnabled() }); +} + +/** + * #7423: adoption keeps routing only for a startup that will re-publish it. A user who has + * since turned the Codex integration off gets the ordinary recovery, even with a stale + * handover receipt on disk; an unreadable config is treated the same way. + */ +function startupSyncEnabled(): boolean { + try { return shouldSyncCodexOnStart(loadConfig()); } catch { return false; } } diff --git a/src/cli/index.ts b/src/cli/index.ts index 9b59169f38d..5fdbac8685d 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -42,7 +42,8 @@ import { type RetainedCodexProviderTable, } from "../codex/inject/restore"; import { stripGrokConfig } from "../grok/inject"; -import { STOP_HISTORY_DEFERRED_EXIT_CODE, STOP_HISTORY_INCOMPLETE_EXIT_CODE } from "../update/stop-contract.mjs"; +import { STOP_HISTORY_DEFERRED_EXIT_CODE, STOP_HISTORY_INCOMPLETE_EXIT_CODE, STOP_TEARDOWN_HANDED_OVER_EXIT_CODE } from "../update/stop-contract.mjs"; +import { announceStopHandover, completeUpdateTeardownHandover, forgetUpdateTeardownHandover, stopHandsTeardownOver, updateTeardownHandoverRequested } from "../update/teardown-handover"; import { describeHistoryJobFailure, resolveCodexHistoryJobTarget, @@ -143,7 +144,7 @@ import { StartOwnershipRollbackUncertainError, } from "./start-ownership-publication"; import { syncModelsToCodex } from "../codex/sync"; -import { localClientSkipReason, shouldSyncGrokOnStart, syncCodexOnStartIfEnabled } from "../codex/desired-state"; +import { localClientSkipReason, shouldSyncCodexOnStart, shouldSyncGrokOnStart, syncCodexOnStartIfEnabled } from "../codex/desired-state"; import { honorSiblingMarker, markSiblingStart, siblingOfLivePort, siblingRuntimeField, siblingStopFoundOwner, withoutSiblingMarker } from "../codex/sibling-start"; import { consumeSiblingHandoff } from "../codex/sibling-handoff"; import { @@ -384,6 +385,7 @@ async function findProxyOwnerBeforeJournalRecovery( } async function handleStart(options: { block?: boolean } = {}) { + forgetUpdateTeardownHandover(); // #7423: a recovery start must not pass the marker to its update workers updateRestartChild?.check(); // A supervised service child defers to a foreign recorded owner before doing // anything else. 'ocx service start' refuses this activation path already, but @@ -752,6 +754,8 @@ async function handleStart(options: { block?: boolean } = {}) { }, ); if (!startupSync.ran) console.log(startupLeftCodexNativeLine(localClientSkipReason(config), server.port ?? port)); + // #7423: routing an update kept in place for this proxy is ours once the startup sync published. + if (!siblingStart && startupSync.published) completeUpdateTeardownHandover(pid => isProcessAlive(pid) && isLikelyOcxProcess(pid)); await refreshOwnedRaycastCatalog(config, port); if (!cleaned && !shuttingDown) healthSentinel = startServiceHealthSentinel(config); // after startup reconciliation; never mid-boot or mid-drain // #1046: one warning per startup, after BOTH writes. The server's cache @@ -1130,11 +1134,14 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) { const inheritedTeardowns = listPendingTeardowns() .filter(read => isPendingTeardownAbandoned(read, teardownOwnerStillRunning)); let teardownNonce: string | undefined; + // #7423: an update that restarts the proxy asks this stop to leave Codex routing for the replacement. + const handoverRequested = updateTeardownHandoverRequested(process.env, () => shouldSyncCodexOnStart(loadConfig())) && !stoppingSibling && !connectedClientSkip; + let deferralHonoredByProxy = false; const claimTeardown = (endpoint: { hostname: string; port: number }, endpointSource: "exact" | "guessed") => { if (teardownNonce || stoppingSibling) return; if (connectedClientSkip) return; try { - teardownNonce = claimPendingTeardown(endpoint, endpointSource).nonce; + teardownNonce = claimPendingTeardown(endpoint, endpointSource, process.pid, handoverRequested ? "update-restart" : undefined).nonce; } catch (err) { // Without a receipt the proxy performs its own teardown, which is the pre-#3008 // behaviour: correct for every backend that cannot respawn, and merely early for @@ -1161,12 +1168,15 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) { // A valid receipt means the proxy deferred shared teardown to this process. If the // receipt could not be written, the proxy restores it itself and the caller must not // attempt a second restore after a graceful stop. A hard-kill always leaves restore - // to this process. + // to this process. #7423: only a confirmed deferral kept the proxy's own exit cleanup off. + if (graceful && teardownNonce) deferralHonoredByProxy = true; return graceful && !teardownNonce; }; const approvedEndpoint = snapshot ? { hostname: probeHostname(snapshot.approval.hostname || undefined), port: snapshot.approval.port } : null; + // Only an exact runtime endpoint may back a handover; without one the ordinary restore runs. + const claimHandoverBeforeManager = () => { const exact = handoverRequested ? endpointOf(readRuntimePort()) : null; if (exact) claimTeardown(exact, "exact"); }; let guardedStep: Awaited> | null = null; try { const serviceStop = snapshot @@ -1182,7 +1192,8 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) { })).service // A sibling never runs under a service manager: an installed one belongs to the live owner, // and its ownership check would fail this stop and skip the stale-record purge below. - : stoppingSibling || connectedClientAlreadyStopped ? "absent" : stopServiceIfInstalledDetailed(); + // #7423: a manager stop takes the proxy down itself, so a handover claims its receipt first. + : stoppingSibling || connectedClientAlreadyStopped ? "absent" : (claimHandoverBeforeManager(), stopServiceIfInstalledDetailed()); record.service = serviceStop; stoppedService = serviceStop === "stopped" || serviceStop === "stopped-respawnable"; schedulerCanRespawn = serviceStop === "stopped-respawnable"; @@ -1425,7 +1436,10 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) { if (stoppingSibling) console.log(`↩️ Client routing stays on the proxy at port ${siblingOfPort}; this sibling had no shared teardown to run.`); if (connectedClientSkip) console.log(connectedClientSkip); const restoreBlocked = ownershipBlocked || inheritedBlocks || nativeRestoreHandledByProxy || stoppingSibling || connectedClientSkip !== null; - if (!restoreBlocked) { + const handedOver = stopHandsTeardownOver({ requested: handoverRequested, restoreBlocked, nonce: teardownNonce, + stopFailed, recoveredCount: recoveredNonces.length, proxy: record.proxy, service: record.service, deferralHonored: deferralHonoredByProxy }); + if (handedOver) announceStopHandover(); + else if (!restoreBlocked) { if (recoveredNonces.length > 0) { // A previous deferred stop died before restoring, and the probe says its endpoint is // not answering. That is the whole point of leaving the receipt behind. @@ -1503,6 +1517,7 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) { ? STOP_HISTORY_DEFERRED_EXIT_CODE : 1; } + else if (handedOver) process.exitCode = STOP_TEARDOWN_HANDED_OVER_EXIT_CODE; const signals = { failed: stopFailed, historyOnly: historyOnlyFailure, diff --git a/src/codex/desired-state.ts b/src/codex/desired-state.ts index c9b29d083c8..f9fee1abd77 100644 --- a/src/codex/desired-state.ts +++ b/src/codex/desired-state.ts @@ -258,7 +258,7 @@ export async function syncCodexOnStartIfEnabled( config: LocalClientSyncConfig, sync: CodexStartupSync = defaultStartupSync, readinessGate?: ReadinessGate, -): Promise<{ ran: boolean; catalogWritten: boolean; cacheSynced: boolean }> { +): Promise<{ ran: boolean; catalogWritten: boolean; cacheSynced: boolean; published?: boolean }> { if (!shouldSyncCodexOnStart(config)) { // The user explicitly turned Codex off: there is nothing to sync, so the // proxy is ready as soon as it is up. The gate is driven here so /readyz @@ -277,10 +277,15 @@ export async function syncCodexOnStartIfEnabled( const outcome = readinessGate ? await runStartupReadinessSync(readinessGate, async () => (await sync(port)) ?? null) : await sync(port).catch(() => undefined); + // #7423: "published" means this proxy now owns what Codex reads: the sync succeeded and + // wrote config and catalog (or the catalog, when an external provider owns config.toml). + // A skipped sync (sibling, hub gate, desired OFF) published nothing of its own. + const status = (outcome as { status?: unknown } | null | undefined)?.status; return { ran: true, catalogWritten: outcome?.catalogWritten === true, cacheSynced: outcome?.cacheSynced === true, + published: outcome?.ok === true && (status === "applied" || status === "catalog-only"), }; } diff --git a/src/codex/journal.ts b/src/codex/journal.ts index 72a4aebe3ca..fb8311a551e 100644 --- a/src/codex/journal.ts +++ b/src/codex/journal.ts @@ -486,6 +486,12 @@ function warnIfJournalRetained(): void { export interface ReconcileJournalOptions { activeClientApiKeyId?: string; + /** + * #7423: an update stopped the previous proxy and handed its Codex routing to this one. + * A process-owned journal whose owner is gone is then adopted rather than replayed, so + * startup does not restore native config in the seconds before its own sync re-injects. + */ + adoptUpdateHandover?: boolean; } export function reconcileJournal(options: ReconcileJournalOptions = {}): boolean { @@ -527,6 +533,15 @@ export function reconcileJournal(options: ReconcileJournalOptions = {}): boolean return false; } } + if (options.adoptUpdateHandover) { + // The original baseline and injected-state hashes stay as written; only ownership moves. + // Published like every other journal write, so a replaced Codex home is refused rather + // than overwritten; a refusal preserves both the journal and the config. + const adopted = JSON.stringify({ ...journal, pid: process.pid, owner: { kind: "process", pid: process.pid } }); + try { publishCodexArtifact(JOURNAL_PATH, handle, (path, hooks) => atomicWriteFile(path, adopted, undefined, hooks)); } + catch { console.error("⚠️ The Codex journal handed over by an update could not be adopted; the journal and configuration were preserved."); } + return false; + } const restored = restoreJournalState({ heldConfigWriteLock: handle }); if (restored.unverified) { console.error("⚠️ Codex journal recovery was not verified; current configuration files and the journal were preserved."); diff --git a/src/config/pending-teardown.ts b/src/config/pending-teardown.ts index 02228d5b748..5632d0739e5 100644 --- a/src/config/pending-teardown.ts +++ b/src/config/pending-teardown.ts @@ -54,6 +54,15 @@ export type PendingTeardownReceipt = { * guessed receipt therefore fails closed into manual recovery. */ endpointSource: "exact" | "guessed"; + /** + * Set when the stop handed its teardown to a replacement proxy instead of performing it + * (#7423). An update that is about to restart the proxy keeps Codex routing and the + * routed catalog in place across the gap, so a Codex process that loads them while the + * package is replaced still sees the routed rows. The obligation is the same one: the + * replacement discharges it after its first successful publication, and if no + * replacement ever comes up, the next `ocx stop` finishes it like any abandoned receipt. + */ + handover?: "update-restart"; }; /** @@ -111,6 +120,7 @@ function isReceipt(value: unknown, nonce: string): value is PendingTeardownRecei && receipt.nonce === nonce && typeof receipt.createdAt === "string" && (receipt.endpointSource === "exact" || receipt.endpointSource === "guessed") + && (receipt.handover === undefined || receipt.handover === "update-restart") && endpointOk; } @@ -119,11 +129,15 @@ export function claimPendingTeardown( endpoint: { hostname: string; port: number }, endpointSource: "exact" | "guessed", ownerPid: number = process.pid, + handover?: "update-restart", ): PendingTeardownReceipt { const dir = getConfigDir(); assertNotRealHomeUnderTest(dir); const nonce = randomBytes(16).toString("hex"); - const receipt: PendingTeardownReceipt = { ownerPid, nonce, createdAt: new Date().toISOString(), endpoint, endpointSource }; + const receipt: PendingTeardownReceipt = { + ownerPid, nonce, createdAt: new Date().toISOString(), endpoint, endpointSource, + ...(handover ? { handover } : {}), + }; atomicWriteFile(pendingTeardownPathFor(nonce), JSON.stringify(receipt, null, 2) + "\n"); return receipt; } @@ -323,3 +337,10 @@ export function deferralMatchesReceipt(nonce: string | null): boolean { if (!nonce || !NONCE_RE.test(nonce)) return false; return readPendingTeardown(nonce).state === "valid"; } + +/** Does this request name a valid obligation that an update handed to its replacement (#7423)? */ +export function isUpdateHandoverReceipt(nonce: string | null): boolean { + if (!nonce || !NONCE_RE.test(nonce)) return false; + const read = readPendingTeardown(nonce); + return read.state === "valid" && read.receipt.handover === "update-restart"; +} diff --git a/src/server/stop-teardown.ts b/src/server/stop-teardown.ts index d31e2b5dd19..515743e73a0 100644 --- a/src/server/stop-teardown.ts +++ b/src/server/stop-teardown.ts @@ -23,6 +23,10 @@ export function stopCapabilities() { export type StopTeardownIo = { /** Does the nonce this request carries name a readable obligation on disk? */ ownsReceipt?: (nonce: string | null) => boolean; + /** Does that obligation hand the teardown to the proxy an update restarts (#7423)? */ + isHandoverReceipt?: (nonce: string | null) => boolean; + /** Keep Codex, Grok and env fences in place when this process exits. */ + keepClientConfigOnExit?: () => void; restoreNativeCodex?: () => Promise; stripGrok?: () => GrokStripResult; }; @@ -96,6 +100,22 @@ export async function performStopTeardown(url: URL, io: StopTeardownIo = {}): Pr } const ownsReceipt = io.ownsReceipt ?? deferralMatchesReceipt; if (deferralHonored(url, ownsReceipt)) { + // A handover outlives this process: the exit cleanup would otherwise restore native + // Codex and put the pristine catalog back right after the stop kept them (#7423). + // Exiting as a recycle leaves them for the replacement, as a dashboard restart does. + const nonce = url.searchParams.get("teardownNonce"); + const handover = io.isHandoverReceipt + ? io.isHandoverReceipt(nonce) + : (await import("../config/pending-teardown")).isUpdateHandoverReceipt(nonce); + if (handover) { + if (io.keepClientConfigOnExit) io.keepClientConfigOnExit(); + else (await import("./lifecycle")).markRecyclingForExit(); + return { + success: true, + message: "Proxy stopping; Codex routing and catalog kept for the updated proxy.", + sharedTeardown: "deferred", + }; + } // Not "native Codex restored": nothing was restored here, and claiming otherwise // would be a success message the operator cannot verify. return { diff --git a/src/update/bind-probe.ts b/src/update/bind-probe.ts new file mode 100644 index 00000000000..016402abb9f --- /dev/null +++ b/src/update/bind-probe.ts @@ -0,0 +1,29 @@ +import { spawnSync } from "node:child_process"; + +/** + * Strict bind script: exit 0 only after listen+close. Any listen error (including + * Windows ghost-TCB failures under Bun) is busy — matches published `ocx start` + * probes that treat every listen error as unavailable. + */ +export function strictBindProbeScript(port: number, hostname: string): string { + return [ + "const net=require('net');", + "const s=net.createServer();", + "s.once('error',()=>process.exit(2));", + `s.listen(${Math.trunc(port)},${JSON.stringify(hostname)},()=>s.close(()=>process.exit(0)));`, + "setTimeout(()=>process.exit(3),2500);", + ].join(""); +} + +export function spawnBindProbe(bin: string, script: string): boolean { + try { + const r = spawnSync(bin, ["-e", script], { + windowsHide: true, + timeout: 4000, + stdio: "ignore", + }); + return r.status === 0; + } catch { + return false; + } +} diff --git a/src/update/job.ts b/src/update/job.ts index c9773fb8b92..890a9ef46da 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -53,6 +53,8 @@ import { } from "./npm-cache-preflight.mjs"; import { guiUpdateWorkerCommand } from "./worker-launch"; import { withoutSiblingMarker } from "../codex/sibling-start"; +import { spawnBindProbe, strictBindProbeScript } from "./bind-probe"; +import { withoutUpdateTeardownHandover, withUpdateTeardownHandover } from "./teardown-handover"; import type { WorkerLaunchContext } from "./worker-launch"; const RELEASE_NOTES_URL = "https://github.com/lidge-jun/opencodex/releases/latest"; @@ -140,34 +142,6 @@ function usesNodeLauncher(installer: Installer): boolean { return installer === "npm" || installer === "pnpm"; } -/** - * Strict bind script: exit 0 only after listen+close. Any listen error (including - * Windows ghost-TCB failures under Bun) is busy — matches published `ocx start` - * probes that treat every listen error as unavailable. - */ -function strictBindProbeScript(port: number, hostname: string): string { - return [ - "const net=require('net');", - "const s=net.createServer();", - "s.once('error',()=>process.exit(2));", - `s.listen(${Math.trunc(port)},${JSON.stringify(hostname)},()=>s.close(()=>process.exit(0)));`, - "setTimeout(()=>process.exit(3),2500);", - ].join(""); -} - -function spawnBindProbe(bin: string, script: string): boolean { - try { - const r = spawnSync(bin, ["-e", script], { - windowsHide: true, - timeout: 4000, - stdio: "ignore", - }); - return r.status === 0; - } catch { - return false; - } -} - /** * Live global package Bun — not the npm rename tree the update worker may still * be executing from (`@bitkyc08/.opencodex-*`). Reject the tiny postinstall @@ -700,12 +674,14 @@ function runLoggedCommand( bin: string, args: string[], timeout: number, + env?: NodeJS.ProcessEnv, ): { status: number | null; signal: NodeJS.Signals | null; timedOut: boolean } { job = updateJob(job, {}, `$ ${formatCommand(bin, args)}`); const result = spawnSync(bin, args, { encoding: "utf8", timeout, windowsHide: true, + ...(env ? { env } : {}), }); const stdout = typeof result.stdout === "string" ? result.stdout.trim() : ""; const stderr = typeof result.stderr === "string" ? result.stderr.trim() : ""; @@ -1812,6 +1788,7 @@ export interface GuiUpdateWorkerIo { bin: string, args: string[], timeout: number, + env?: NodeJS.ProcessEnv, ) => { status: number | null; signal: NodeJS.Signals | null }; } @@ -1933,7 +1910,11 @@ export async function runGuiUpdateWorker( - 대안 분석: (1) 서버에서 runUpdate 직접 호출: process.exit/stdio/실행 파일 교체 위험. (2) GUI에서 CLI 명령 안내만 제공: 자동 업데이트 UX 부족. (3) 숨은 worker가 Node launcher/Bun 전역 명령을 실행: 상태 추적과 안전한 재시작이 가능. - 선택 근거: 현재 CLI의 package-manager self-update 우회를 재사용하면서도 GUI 서버 요청 생명주기와 설치 작업을 분리할 수 있어 가장 안정적이다. */ - const result = (io.runCommandFn ?? runLoggedCommand)(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS); + // #7423: this job restarts the proxy itself, so the update's stop hands Codex routing + // and the routed catalog to the replacement instead of restoring native Codex for the + // seconds in between. Without a restart the stop restores, even under an inherited marker. + const updateEnv = restart ? withUpdateTeardownHandover(process.env) : withoutUpdateTeardownHandover(process.env); + const result = (io.runCommandFn ?? runLoggedCommand)(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS, updateEnv); if (result.status !== 0) { if (check.installer === "pnpm") { const verifiedLauncher = pnpmOwner diff --git a/src/update/stop-contract.d.mts b/src/update/stop-contract.d.mts index 1efb77c42d7..f0740172b23 100644 --- a/src/update/stop-contract.d.mts +++ b/src/update/stop-contract.d.mts @@ -1,3 +1,4 @@ /** Declaration for the plain-ESM stop contract shared with `bin/ocx.mjs`. */ export declare const STOP_HISTORY_INCOMPLETE_EXIT_CODE: 79; export declare const STOP_HISTORY_DEFERRED_EXIT_CODE: 80; +export declare const STOP_TEARDOWN_HANDED_OVER_EXIT_CODE: 81; diff --git a/src/update/stop-contract.mjs b/src/update/stop-contract.mjs index b86018ea0d7..bf7e38da08f 100644 --- a/src/update/stop-contract.mjs +++ b/src/update/stop-contract.mjs @@ -32,3 +32,17 @@ export const STOP_HISTORY_INCOMPLETE_EXIT_CODE = 79; * dispatcher already emit. */ export const STOP_HISTORY_DEFERRED_EXIT_CODE = 80; + +/** + * The exit code `ocx stop` uses to say "the proxy is down and its shared teardown was + * handed to the proxy this update is about to start" (#7423). + * + * Only an update that will restart the proxy asks for this. Restoring native Codex here + * replaced the routed catalog with the pristine one, and a Codex process that loaded it + * before the new proxy's first publication kept treating routed models as unknown until + * it was restarted. Like 80, the receipt stays outstanding, and the stop proved before + * choosing this code that it is the only obligation left; unlike 80, nothing refused the + * restore. The replacement discharges the receipt after it publishes, and a stop finishes + * it if no replacement ever starts. + */ +export const STOP_TEARDOWN_HANDED_OVER_EXIT_CODE = 81; diff --git a/src/update/stop-decision.d.mts b/src/update/stop-decision.d.mts index 13f1cd93caf..5fe7fab572f 100644 --- a/src/update/stop-decision.d.mts +++ b/src/update/stop-decision.d.mts @@ -6,5 +6,5 @@ export declare function decidePostStopUpdate(input: { teardownOutstanding?: boolean; }): { proceed: boolean; - reason: "stop-failed" | "runtime-state" | "teardown-outstanding" | "proxy-live" | "proxy-unknown" | "history-only" | "history-deferred" | "ok"; + reason: "stop-failed" | "runtime-state" | "teardown-outstanding" | "proxy-live" | "proxy-unknown" | "history-only" | "history-deferred" | "handed-over" | "ok"; }; diff --git a/src/update/stop-decision.mjs b/src/update/stop-decision.mjs index 0cc1a71a593..3c021a63a41 100644 --- a/src/update/stop-decision.mjs +++ b/src/update/stop-decision.mjs @@ -1,4 +1,8 @@ -import { STOP_HISTORY_DEFERRED_EXIT_CODE, STOP_HISTORY_INCOMPLETE_EXIT_CODE } from "./stop-contract.mjs"; +import { + STOP_HISTORY_DEFERRED_EXIT_CODE, + STOP_HISTORY_INCOMPLETE_EXIT_CODE, + STOP_TEARDOWN_HANDED_OVER_EXIT_CODE, +} from "./stop-contract.mjs"; /** * May an update replace package files after `ocx stop` returned? @@ -29,15 +33,21 @@ import { STOP_HISTORY_DEFERRED_EXIT_CODE, STOP_HISTORY_INCOMPLETE_EXIT_CODE } fr * gate still applies: runtime records and a live or unreadable endpoint abort exactly * as they do for a clean stop, because package replacement under a live server is the * danger this function exists to prevent, and a history refusal says nothing about it. + * - `handed-over` — proceed; the stop is down and kept its receipt on purpose so the + * proxy this update restarts can take the routing over without a native gap (#7423). + * It is gated exactly like `history-deferred`: the child proved its receipt is the only + * obligation, and runtime records or a live or unreadable endpoint still abort. */ export function decidePostStopUpdate({ status, hasRuntimeState, liveness, teardownOutstanding = false }) { const historyOnly = status === STOP_HISTORY_INCOMPLETE_EXIT_CODE; const historyDeferred = status === STOP_HISTORY_DEFERRED_EXIT_CODE; - if (status !== 0 && !historyOnly && !historyDeferred) return { proceed: false, reason: "stop-failed" }; + const handedOver = status === STOP_TEARDOWN_HANDED_OVER_EXIT_CODE; + if (status !== 0 && !historyOnly && !historyDeferred && !handedOver) return { proceed: false, reason: "stop-failed" }; if (hasRuntimeState) return { proceed: false, reason: "runtime-state" }; - if (teardownOutstanding && !historyDeferred) return { proceed: false, reason: "teardown-outstanding" }; + if (teardownOutstanding && !historyDeferred && !handedOver) return { proceed: false, reason: "teardown-outstanding" }; if (liveness === "live") return { proceed: false, reason: "proxy-live" }; if (liveness !== "dead") return { proceed: false, reason: "proxy-unknown" }; if (historyDeferred) return { proceed: true, reason: "history-deferred" }; + if (handedOver) return { proceed: true, reason: "handed-over" }; return { proceed: true, reason: historyOnly ? "history-only" : "ok" }; } diff --git a/src/update/teardown-handover.ts b/src/update/teardown-handover.ts new file mode 100644 index 00000000000..65ec682e9c6 --- /dev/null +++ b/src/update/teardown-handover.ts @@ -0,0 +1,166 @@ +import { + clearPendingTeardown, + isPendingTeardownAbandoned, + listPendingTeardowns, + pendingTeardownPathFor, + pendingTeardownsAreExactly, + readPendingTeardown, +} from "../config/pending-teardown"; +import { isLikelyOcxProcess } from "../config/process-state"; +import { isProcessAlive } from "../lib/process-control"; + +/** + * Update teardown handover (#7423). + * + * An update stops the proxy before replacing the package. That stop used to restore native + * Codex, which puts the pristine catalog back and removes every routed row. The new proxy + * appends them again when it starts, but a Codex process that loads the catalog in between + * keeps treating routed models as unknown until it is restarted. On a long-context model + * that silent fallback metadata turned into a compaction loop. + * + * When the caller is going to restart the proxy, it sets this variable on the update + * child. The `ocx stop` that the update runs then keeps its teardown receipt instead of + * restoring, and the proxy that honours the deferral keeps its client config on exit. + * Ownership stays explicit: the receipt is the same durable obligation any deferred stop + * writes, so a replacement that never starts leaves it for the next `ocx stop` to finish. + * + * An ordinary `ocx stop` never sees the variable and still restores native routing. + */ +export const UPDATE_TEARDOWN_HANDOVER_ENV = "OCX_UPDATE_TEARDOWN_HANDOVER"; + +/** + * `publishesOnStart` answers whether the restarted proxy will run its startup Codex sync. + * Without one it can never discharge the receipt, so the stop restores as usual; an + * unreadable answer is treated the same way. + */ +export function updateTeardownHandoverRequested( + env: NodeJS.ProcessEnv = process.env, + publishesOnStart: () => boolean = () => true, +): boolean { + if (env[UPDATE_TEARDOWN_HANDOVER_ENV] !== "1") return false; + try { return publishesOnStart(); } catch { return false; } +} + +/** An identity-aware "is that stop still running", the rule handleStop uses for receipts. */ +export function teardownOwnerStillRunning(pid: number): boolean { + return isProcessAlive(pid) && isLikelyOcxProcess(pid); +} + +/** + * Is an update handover waiting for this starting proxy? True when a valid, exact handover + * receipt exists whose stop has exited: that stop kept Codex routing for the replacement. + */ +export function updateTeardownHandoverPending(ownerStillRunning: (pid: number) => boolean = teardownOwnerStillRunning): boolean { + return listPendingTeardowns().some(read => read.state === "valid" + && read.receipt.handover === "update-restart" && read.receipt.endpointSource === "exact" + && isPendingTeardownAbandoned(read, ownerStillRunning)); +} + +/** The environment for an update child whose caller will restart the proxy afterwards. */ +export function withUpdateTeardownHandover(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + return { ...env, [UPDATE_TEARDOWN_HANDOVER_ENV]: "1" }; +} + +/** The environment for an update child that must restore as usual, whatever this process inherited. */ +export function withoutUpdateTeardownHandover(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + const child = { ...env }; + delete child[UPDATE_TEARDOWN_HANDOVER_ENV]; + return child; +} + +/** + * Drop the marker from a starting proxy. The updater lanes pass their environment to the + * recovery and replacement starts they spawn; a proxy that kept it would hand it to every + * dashboard update worker it launches later, including ones that never restart. + */ +export function forgetUpdateTeardownHandover(env: NodeJS.ProcessEnv = process.env): void { + delete env[UPDATE_TEARDOWN_HANDOVER_ENV]; +} + +/** The facts `ocx stop` has already settled when it reaches the shared restore. */ +export type StopHandoverFacts = { + requested: boolean; + restoreBlocked: boolean; + nonce: string | undefined; + stopFailed: boolean; + recoveredCount: number; + proxy: string; + /** What the service-manager stop returned; a manager can take the proxy down itself. */ + service?: string; + /** + * The proxy this run stopped answered `deferred` before exiting. Any fallback (a failed + * or unconfirmed /api/stop, then a signal) lets a standalone proxy's exit cleanup restore + * native Codex and the pristine catalog, so nothing is left to hand over. + */ + deferralHonored?: boolean; +}; + +/** + * May this stop skip the restore and hand its teardown to the restarted proxy? + * + * Only when the update asked for it, this run's own receipt is the single obligation in + * the home, and the proxy it named is down: stopped by this run, or gone after a clean + * service-manager stop (a launchd/systemd/scheduler child exits with its manager, and its + * own exit cleanup keeps routing because it runs with OCX_SERVICE). A recovered or foreign + * receipt, a blocked restore or a stop that failed on the way all take the ordinary + * restore instead. The exclusivity check is what lets the updater treat the outstanding + * receipt as expected; the updater still probes the endpoint itself before replacing files. + */ +export function stopHandsTeardownOver(facts: StopHandoverFacts): boolean { + const managerStopped = facts.service === "stopped" || facts.service === "stopped-respawnable"; + const proxyDown = ((facts.proxy === "stopped" || facts.proxy === "stopped-orphan") && facts.deferralHonored === true) + || (facts.proxy === "not-running" && managerStopped); + return facts.requested && !facts.restoreBlocked && facts.nonce !== undefined && !facts.stopFailed + && facts.recoveredCount === 0 && proxyDown && pendingTeardownsAreExactly([facts.nonce]) + && handoverReceiptIsRecoverable(facts.nonce); +} + +/** + * A handed-over receipt must be one a later `ocx stop` can finish on its own if no + * replacement ever starts. A `guessed` endpoint fails closed into manual recovery there, + * so a stop that could only record the configured address restores instead. + */ +function handoverReceiptIsRecoverable(nonce: string): boolean { + const read = readPendingTeardown(nonce); + return read.state === "valid" && read.receipt.endpointSource === "exact" && read.receipt.handover === "update-restart"; +} + +export function announceStopHandover(log: Pick = console): void { + log.log("↪️ Codex routing, its catalog and the Grok config stay in place for the updated proxy, which releases them after its first sync."); + log.log(" If it does not start, run 'ocx start' to take them over or 'ocx stop' to restore native Codex."); +} + +/** + * Release handover receipts once this proxy has published its own routing. + * + * Called after a successful startup sync, when Codex config and catalog point at the + * running proxy again, so the restore the stop handed over is no longer owed. Only a + * handover receipt whose owning stop has exited is released: an ordinary deferred stop + * still has to run its own restore, and a stop that is still running owns its receipt. + */ +export function dischargeUpdateTeardownHandovers(ownerStillRunning: (pid: number) => boolean): { + discharged: string[]; + failed: string[]; +} { + const discharged: string[] = []; + const failed: string[] = []; + for (const read of listPendingTeardowns()) { + if (read.state !== "valid" || read.receipt.handover !== "update-restart") continue; + if (!isPendingTeardownAbandoned(read, ownerStillRunning)) continue; + if (clearPendingTeardown(read.receipt.nonce)) discharged.push(read.receipt.nonce); + else failed.push(read.receipt.nonce); + } + return { discharged, failed }; +} + +/** Startup half: after this proxy published its own routing, release what the update kept. */ +export function completeUpdateTeardownHandover( + ownerStillRunning: (pid: number) => boolean, + log: Pick = console, +): void { + const handover = dischargeUpdateTeardownHandovers(ownerStillRunning); + if (handover.discharged.length > 0) log.log(" Took over the Codex routing an update kept in place."); + for (const nonce of handover.failed) { + log.error(`⚠️ Could not remove the update handover receipt ${pendingTeardownPathFor(nonce)}; the next 'ocx stop' finishes it.`); + } +} diff --git a/structure/runtime.md b/structure/runtime.md index ebeac303a54..35e50832d4e 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -59,7 +59,7 @@ The [CLI management contract](cli-management.md#cli-readiness-diagnostics) owns `ocx resolve` (`src/cli/resolve.ts`) is the machine surface a desktop shell asks instead of resolving the config home, the port, and liveness itself: the home comes from `src/config/paths.ts`, the effective port is the live listener's when the identity-checked `findLiveProxy` answers and the configured `config.port ?? 10100` otherwise, and the liveness verdict is that same module's output (pid, runtime-versus-config provenance, version, role). Config reads go through `readConfigDiagnostics`, not `loadConfig`: a missing file is defaults, but an invalid file exits 1 instead of being repaired to defaults, because a defaulted port is a guess the caller must refuse. Liveness is three-valued: when `findLiveProxy` returns null, resolve re-asks the endpoints with `endpointsToProve` and `everyEndpointProvenDown`, using the in-process `probeEndpointLiveness` from `src/server/proxy-liveness.ts`. Only unanimous definitive "dead" becomes `absent-proven`; a timeout or reset stays unknown, exits 1 and never authorises a start. The transient exclusive-bind fallback in `src/update/proxy-liveness-probe.mjs` belongs only to the npm/Bun updater. Resolve stays in-process because a compiled standalone binary cannot run a JavaScript probe with `execPath -e`. Discovery borrows `START_OWNERSHIP_LIVENESS`, the start path's ownership budget — the verdict feeds the shell's launch decision, so the cost of a false "nobody listening" is the duplicate proxy (#5004). The verb is in `skipsCodexShimAutoRestore`, so a read-only lookup never triggers a shim repair. Arguments are pre-parsed in `src/cli/root.ts` and exit 64 before any preflight side effect, the same ordering `ocx ready` obeys. `ocx-resolve/1` also reports `ownership` (`none`, `owned`, or `unknown`) and `takeover` (`supported` with protocol version, minimum CLI version and compatibility token, or `blocked` with a reason). A known liveness verdict still exits 0 when ownership is unknown, but takeover is blocked. An unreadable second service-state read blocks takeover instead of becoming an absent registration. `src/service/managing-cli.ts` observes the registered and selected PATH CLIs, including Windows PATHEXT order and file validation, before compatibility is offered. A `--version` probe is retried once on Windows — one transient failure must not read as an unobservable CLI — and a Windows `.CMD`/`.BAT` shim is invoked through a single verbatim-quoted `cmd /c` line, so a spaced executable path and spaced arguments both survive cmd's re-parse. `src/cli/route-policy.ts` distinguishes a missing profile (exit 4) from malformed `show` arguments (exit 2); an upstream failure propagates through the management error mapping. -`ocx stop --json` is a reporting layer over the unchanged stop path. `src/cli/index.ts` threads a `StopRunRecord` through the existing receipt, drain, respawn-verification and restore flow, and `src/cli/stop-report.ts` maps the recorded facts plus the signals that already pick the exit code into one versioned document (`schema: "ocx-stop/1"`). With `--json` the human lines print on stderr and stdout carries only that document; exit codes 0/1/79/80 cross the process boundary unchanged. The desktop's opt-in guarded stop adds exact approved PID, endpoint, home, CLI version and compatibility-token expectations; plain `ocx stop` retains its ordinary behavior. Under the ownership mutation lease, `src/cli/stop-approval.ts` re-resolves them and `src/service/guarded-manager-target.ts` binds any installed manager's process tree to the approved PID before a stop action. A pre-action mismatch returns `approval-changed` without stopping, and the summary carries `detail` naming the single guard fact that failed — live PID/port/hostname/CLI drift, an unverifiable ownership claim, a blocked takeover, or the compatibility fingerprint — so a transient re-probe can be told from genuine drift in the caller's log. Immediately before any manager stop or direct PID signal, the CLI rechecks the same manager identity or proven absence; a change also returns `approval-changed`. A non-OpenCodex process replacing the OS job in the remaining instant before the manager command is a residual same-user risk outside the ownership lease. The guarded path stops the bound manager, or the approved PID directly when managers are proven absent, then uses a five-second shared deadline to verify PID exit, port availability and definitive endpoint absence. Only after settlement does a read-only manager probe require launchd `not-loaded` in both domains or systemd `inactive` with `MainPID=0`; the state is checked again before a success summary. Timeout, active or unreadable manager state returns terminal `manager-still-active`, even if the endpoint briefly refuses. Windows managers bind through `src/service/windows-guarded-manager.ts`: a Task Scheduler backend must prove a recognized OpenCodex registration against the bounded current-user SID/name lookup, a running task, and a wrapper ancestor carrying this home's canonical paths — presence and running state are read root-folder-scoped because every `/tn` operation resolves a bare name there and nowhere else, so a same-named task inside a task folder is a different registration neither the probe nor the stop can reach; the registered definition itself is read only on the running path, and a state or registration read that cannot answer re-probes presence so a task deleted between probes — which can respawn nothing — reads absent while a merely unreadable one stays unknown; an unreadable current identity leaves the manager unknown; a WinSW backend must prove this installation's binary and SCM service PID ancestry; either marks `childNeedsSeparateStop`, so the approved PID receives its own graceful signal only after a successful manager stop because `schtasks /end` does not reliably cascade to the wrapper's child (#764); a failed or unknown manager stop leaves the child untouched. Post-stop Windows inactivity means the task not running, no surviving wrapper process, and no started WinSW — never mere registration presence; a running-state read that cannot answer re-probes presence the same way, so a task deleted between probes reports inactive while a persisting one stays unknown. Every Windows manager command a verdict or stop depends on — schtasks, sc.exe, WinSW verbs — runs under a bounded `execFileSync` timeout so a wedged Task Scheduler or Service Control Manager cannot hang the guarded stop; a killed command classifies through the same fail-closed paths a nonzero exit would (the interactive `install /p` password prompt is the one deliberate exception). A `wscript.exe`, `cscript.exe` or `cmd.exe` whose command line CIM cannot read makes the state unprovable only when it is tied to the proxy: pre-stop, as an ancestor of the approved PID; post-stop, as the former bound manager process or a descendant of it. Unrelated unreadable shells, which a non-admin query returns for other users' and elevated processes, are not evidence either way. An unprovable Windows manager still blocks guarded takeover. The token checks snapshot consistency, not whether a person approved the desktop prompt. `src/cli/stop-restore.ts` owns the shared Codex/Grok restoration and its history-only, history-deferred, and other failure categories. +`ocx stop --json` is a reporting layer over the unchanged stop path. `src/cli/index.ts` threads a `StopRunRecord` through the existing receipt, drain, respawn-verification and restore flow, and `src/cli/stop-report.ts` maps the recorded facts plus the signals that already pick the exit code into one versioned document (`schema: "ocx-stop/1"`). With `--json` the human lines print on stderr and stdout carries only that document; exit codes 0/1/79/80/81 cross the process boundary unchanged. The desktop's opt-in guarded stop adds exact approved PID, endpoint, home, CLI version and compatibility-token expectations; plain `ocx stop` retains its ordinary behavior. Under the ownership mutation lease, `src/cli/stop-approval.ts` re-resolves them and `src/service/guarded-manager-target.ts` binds any installed manager's process tree to the approved PID before a stop action. A pre-action mismatch returns `approval-changed` without stopping, and the summary carries `detail` naming the single guard fact that failed — live PID/port/hostname/CLI drift, an unverifiable ownership claim, a blocked takeover, or the compatibility fingerprint — so a transient re-probe can be told from genuine drift in the caller's log. Immediately before any manager stop or direct PID signal, the CLI rechecks the same manager identity or proven absence; a change also returns `approval-changed`. A non-OpenCodex process replacing the OS job in the remaining instant before the manager command is a residual same-user risk outside the ownership lease. The guarded path stops the bound manager, or the approved PID directly when managers are proven absent, then uses a five-second shared deadline to verify PID exit, port availability and definitive endpoint absence. Only after settlement does a read-only manager probe require launchd `not-loaded` in both domains or systemd `inactive` with `MainPID=0`; the state is checked again before a success summary. Timeout, active or unreadable manager state returns terminal `manager-still-active`, even if the endpoint briefly refuses. Windows managers bind through `src/service/windows-guarded-manager.ts`: a Task Scheduler backend must prove a recognized OpenCodex registration against the bounded current-user SID/name lookup, a running task, and a wrapper ancestor carrying this home's canonical paths — presence and running state are read root-folder-scoped because every `/tn` operation resolves a bare name there and nowhere else, so a same-named task inside a task folder is a different registration neither the probe nor the stop can reach; the registered definition itself is read only on the running path, and a state or registration read that cannot answer re-probes presence so a task deleted between probes — which can respawn nothing — reads absent while a merely unreadable one stays unknown; an unreadable current identity leaves the manager unknown; a WinSW backend must prove this installation's binary and SCM service PID ancestry; either marks `childNeedsSeparateStop`, so the approved PID receives its own graceful signal only after a successful manager stop because `schtasks /end` does not reliably cascade to the wrapper's child (#764); a failed or unknown manager stop leaves the child untouched. Post-stop Windows inactivity means the task not running, no surviving wrapper process, and no started WinSW — never mere registration presence; a running-state read that cannot answer re-probes presence the same way, so a task deleted between probes reports inactive while a persisting one stays unknown. Every Windows manager command a verdict or stop depends on — schtasks, sc.exe, WinSW verbs — runs under a bounded `execFileSync` timeout so a wedged Task Scheduler or Service Control Manager cannot hang the guarded stop; a killed command classifies through the same fail-closed paths a nonzero exit would (the interactive `install /p` password prompt is the one deliberate exception). A `wscript.exe`, `cscript.exe` or `cmd.exe` whose command line CIM cannot read makes the state unprovable only when it is tied to the proxy: pre-stop, as an ancestor of the approved PID; post-stop, as the former bound manager process or a descendant of it. Unrelated unreadable shells, which a non-admin query returns for other users' and elevated processes, are not evidence either way. An unprovable Windows manager still blocks guarded takeover. The token checks snapshot consistency, not whether a person approved the desktop prompt. `src/cli/stop-restore.ts` owns the shared Codex/Grok restoration and its history-only, history-deferred, and other failure categories. `ocx service status --json` (`src/service/status-json.ts`) is the passive machine form of the service report, read by peers such as GOAT's device inventory. `src/service/cli.ts` accepts `--json` for `status` only (`planServiceCommand` refuses it on every other verb before any installation probe) and answers it before `platformOps()` resolves, so no install, start, stop, repair or migration path is reachable; `skipsCodexShimAutoRestore` exempts `service status --json` so the CLI preflight repairs no Codex shim first, the collection runs inside `runPassiveRead` so the serving probe's config load neither hardens modes nor backs up a malformed file, and `--json` on any other verb is refused before the Desktop check and the Windows mutation lock; an unsupported platform or Docker gets a document instead of the text path's exit. The document reuses exactly the reads the text report makes — `diagnoseService()` (launchd `print`, systemd observation verbs, Task Scheduler `/query`, SCM/WinSW status) and, only when a registration exists, the 1.5-second loopback `confirmServiceServing` probe — plus the ownership-lease and Desktop login lines. `running` is the manager's verdict (the value `startup.serviceRunning` carries in `ocx status --json`); `serving` reports separately whether a proxy answers on the installed port. `backend` is `launchd`, `systemd`, `schtasks`, `winsw`, or `none` when nothing is registered. The systemd branch of `diagnoseService` derives through the pure `deriveSystemdServiceDiagnostic`, matching the launchd and Windows derivations. `tests/service/service-status-json.test.ts` snapshots each platform branch and records every subprocess entry point on darwin, linux and win32 against a read-only allowlist. `ocx status --json` carries an additive `install` verdict (`src/cli/status-install.ts`) mapped from the updater's own `detectInstallOwnership()`: `method` is `bun`, `npm`, `pnpm`, `mise`, `source`, `standalone` or `unknown` (detector failure), and `evidence.basis` names the package layout, the mise backend record (tool alias and backend only) or the standalone binary without printing a path (`tests/cli/status-install-method.test.ts`). @@ -275,7 +275,7 @@ with no proxy running and a dead owner (#4897). `isLikelyOcxProcess` asks the br `verifyPidIdentity`, without the `start` verb, because a receipt owner is an `ocx stop` or the `ocx update` worker that drove it rather than the proxy. Recognizing a receipt as abandoned only admits it to recovery; a valid receipt must still prove its recorded endpoint is down before -anything is restored, and the package launcher still decides nothing itself. +anything is restored, and the package launcher still decides nothing itself. An update whose dashboard job will restart the proxy runs that stop with `OCX_UPDATE_TEARDOWN_HANDOVER=1` (#7423): the receipt is marked `handover: "update-restart"`, the stop exits 81 without restoring only when that receipt is the single obligation in the home, the old proxy honoring the deferral exits as a recycle so its exit cleanup keeps Codex config and the routed catalog, both updater lanes proceed past exit 81 under the same gates as 80, and the replacement discharges abandoned handover receipts after its first published startup sync (`src/update/teardown-handover.ts`). A replacement that never starts leaves an ordinary abandoned receipt for the next `ocx stop`. While such a receipt is pending, startup journal recovery adopts the previous proxy's process-owned journal instead of replaying native config before the first sync (`reconcileJournal({ adoptUpdateHandover })`). Port reclamation must honor a rejected OCX verifier result even for a PID captured before stop or update. A rejected live holder prevents both termination and TCP-row deletion for that scan; later diff --git a/tests/cli/cli-dispatch.test.ts b/tests/cli/cli-dispatch.test.ts index 544ca90c7fc..17a90be01dd 100644 --- a/tests/cli/cli-dispatch.test.ts +++ b/tests/cli/cli-dispatch.test.ts @@ -632,7 +632,8 @@ describe("a sibling start leaves shared client routing to the live owner", () => // A sibling never runs under a service manager, so an installed one is the live owner's. Asking // it failed the ownership check from the sibling's home: exit 1, and after a hard kill the // stale sibling records were never purged. - expect(stop).toContain(': stoppingSibling || connectedClientAlreadyStopped ? "absent" : stopServiceIfInstalledDetailed();'); + // #7423: the non-sibling branch claims an update-handover receipt first; the sibling term is unchanged. + expect(stop).toContain(': stoppingSibling || connectedClientAlreadyStopped ? "absent" : (claimHandoverBeforeManager(), stopServiceIfInstalledDetailed());'); // The only other caller is the guarded desktop step, which reaches the manager only when it // provably owns the approved pid; a sibling's pid never is. expect(stop.split("stopServiceIfInstalledDetailed()").length - 1).toBe(2); diff --git a/tests/fixtures/handover-journal-probe.ts b/tests/fixtures/handover-journal-probe.ts new file mode 100644 index 00000000000..2e728dd91cb --- /dev/null +++ b/tests/fixtures/handover-journal-probe.ts @@ -0,0 +1,50 @@ +/** #7423 startup journal recovery probe: runs the real reconcileStartupJournal in an isolated child. */ +import { spyOn } from "bun:test"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const [mode, root] = [process.argv[2] as "swap" | "off", process.argv[3]!]; +const codex = join(root, "codex"); +const ocx = join(root, "ocx"); +for (const path of [codex, ocx]) mkdirSync(path, { recursive: true }); +process.env.CODEX_HOME = codex; +process.env.OPENCODEX_HOME = ocx; +if (mode === "off") writeFileSync(join(ocx, "config.json"), JSON.stringify({ providers: {}, clientIntegrations: { codex: false } })); +const original = "# original\nmodel_provider = \"openai\"\n"; +const injected = "# injected\nmodel_provider = \"opencodex\"\n"; +const configPath = join(codex, "config.toml"); +const journalPath = join(codex, "opencodex-journal.json"); +const dead = 999_999_991; +const journal = { + version: 1, originalConfig: Buffer.from(original).toString("base64"), originalProfile: null, + injectedConfigHash: createHash("sha256").update(injected).digest("hex"), injectedProfileHash: null, + pid: dead, owner: { kind: "process", pid: dead }, opencodexHome: ocx, timestamp: new Date().toISOString(), +}; +writeFileSync(configPath, injected); +writeFileSync(journalPath, JSON.stringify(journal)); +const receipts = await import("../../src/config/pending-teardown"); +// A receipt left by an update whose stop exited long ago. +receipts.claimPendingTeardown({ hostname: "127.0.0.1", port: 10100 }, "exact", dead + 1, "update-restart"); +// A replacement home bound to another live owner, installed after the journal was read. +const replacement = { ...journal, originalConfig: Buffer.from("# replacement baseline\n").toString("base64"), pid: process.pid, owner: { kind: "process", pid: process.pid } }; +const kill = process.kill; +const spy = spyOn(process, "kill").mockImplementation(((pid: number, signal?: NodeJS.Signals | number) => { + if ((pid !== dead && pid !== dead + 1) || signal !== 0) return kill.call(process, pid, signal); + if (mode === "swap" && pid === dead) { + renameSync(codex, join(root, "parked-codex")); + mkdirSync(codex); + writeFileSync(configPath, injected); + writeFileSync(journalPath, JSON.stringify(replacement)); + } + throw Object.assign(new Error("dead fixture owner"), { code: "ESRCH" }); +}) as typeof process.kill); +try { + (await import("../../src/cli/cross-home-owner")).reconcileStartupJournal(); +} finally { + spy.mockRestore(); +} +console.log(JSON.stringify(mode === "swap" + ? { replacementPreserved: readFileSync(journalPath, "utf8") === JSON.stringify(replacement) } + : { config: readFileSync(configPath, "utf8"), journalExists: existsSync(journalPath) })); + diff --git a/tests/fixtures/parent-stop-runner.ts b/tests/fixtures/parent-stop-runner.ts index 93ef7fae037..7f67e822a24 100644 --- a/tests/fixtures/parent-stop-runner.ts +++ b/tests/fixtures/parent-stop-runner.ts @@ -25,6 +25,12 @@ const options = JSON.parse(readFileSync(0, "utf8")) as { alreadyStopped?: boolean; absence?: "dead" | "unknown" | "live"; managerState?: "inactive" | "active" | "unknown"; + /** #7423: run the stop the way an update that restarts the proxy runs it. */ + handover?: boolean; + /** The runtime-port record is missing, so the stop can only record a guessed endpoint. */ + noRuntimeRecord?: boolean; + /** Like the real readPid, report no PID once the process is gone (a manager stop exited it). */ + deadPidReadsNull?: boolean; }; if (options.proxyProbePort) { // Deliberately bypass every fixture mock for the real proxy-environment regression. @@ -100,8 +106,8 @@ mock.module("../../src/config/pending-teardown", () => ({ const state = await import("../../src/config/process-state"); mock.module("../../src/config/process-state", () => ({ ...state, - readPid: () => options.orphan || options.alreadyStopped ? null : fakePid, - readRuntimePort: () => options.alreadyStopped ? null : endpoint, + readPid: () => options.orphan || options.alreadyStopped || (options.deadPidReadsNull && !alive) ? null : fakePid, + readRuntimePort: () => options.alreadyStopped || options.noRuntimeRecord ? null : endpoint, removePid() {}, removeRuntimePort() {}, })); @@ -198,6 +204,7 @@ mock.module("../../src/server/direct-local-http", () => ({ ...direct, })); process.argv = [process.execPath, "ocx", "stop", ...(options.json ? ["--json"] : [])]; +if (options.handover) process.env.OCX_UPDATE_TEARDOWN_HANDOVER = "1"; if (options.exitCleanup) { delete process.env.OCX_SERVICE; // Execute the actual startup exit closure, with only host/config effects replaced. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index c7858ea3657..f8c40421f3d 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1283,6 +1283,7 @@ "update-mise-node-runtime.test.ts": "update", "update-pnpm.test.ts": "update", "update-refresh.test.ts": "update", "update-restart-lease.test.ts": "update", "update-stop-classification.test.ts": "update", "update-stop-first.test.ts": "update", + "update-teardown-handover.test.ts": "update", "update-transactional-leftovers.test.ts": "update", "update-transactional.test.ts": "update", "update-tray-handoff.test.ts": "update", "update-tree-ownership.test.ts": "update", "upstream-connect-error.test.ts": "server", "upstream-error-message-fallback.test.ts": "lib", diff --git a/tests/providers/xai/grok-lifecycle.test.ts b/tests/providers/xai/grok-lifecycle.test.ts index 8f1dbcae0a9..34abd0acd9a 100644 --- a/tests/providers/xai/grok-lifecycle.test.ts +++ b/tests/providers/xai/grok-lifecycle.test.ts @@ -333,10 +333,11 @@ describe("Grok fence lifecycle wiring", () => { const updateSource2 = readFileSync(repoPath("src", "update", "index.ts"), "utf8"); expect(updateSource2).toContain("teardownOutstanding: pendingTeardownOutstanding()"); const decisionSource = readFileSync(repoPath("src", "update", "stop-decision.mjs"), "utf8"); - // The gate has exactly one exemption, and it is the child saying it kept those - // receipts on purpose after the Codex history preflight refused (#4718). Anything - // else — including a stop that merely exited 0 — still aborts the install. - expect(decisionSource).toContain('if (teardownOutstanding && !historyDeferred) return { proceed: false, reason: "teardown-outstanding" };'); + // The gate has exactly two exemptions, both the child saying it kept its own receipt on + // purpose: after the Codex history preflight refused (#4718), and when an update hands + // its teardown to the proxy it restarts (#7423). Anything else — including a stop that + // merely exited 0 — still aborts the install. + expect(decisionSource).toContain('if (teardownOutstanding && !historyDeferred && !handedOver) return { proceed: false, reason: "teardown-outstanding" };'); expect(decisionSource).toContain("const historyDeferred = status === STOP_HISTORY_DEFERRED_EXIT_CODE;"); const receiptSource = readFileSync(repoPath("src", "config", "pending-teardown.ts"), "utf8"); expect(receiptSource).toContain('from "./pending-teardown-names.mjs"'); diff --git a/tests/update/update-job.test.ts b/tests/update/update-job.test.ts index 169f3abdde1..2e6c9b16da9 100644 --- a/tests/update/update-job.test.ts +++ b/tests/update/update-job.test.ts @@ -1852,7 +1852,7 @@ describe("immutable update target (WP160)", () => { const cacheGateAt = source.indexOf("const cachePreflight = (io.cachePreflightFn ?? runNpmCachePreflight)();"); const trayStopAt = source.indexOf("handoffWindowsTrayForUpdate(tray"); const failAt = source.indexOf('updateJob(job, { status: "failed", error: integrity.reason });'); - const spawnAt = source.indexOf("const result = (io.runCommandFn ?? runLoggedCommand)(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS);"); + const spawnAt = source.indexOf("const result = (io.runCommandFn ?? runLoggedCommand)(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS, updateEnv);"); expect(gateAt).toBeGreaterThan(-1); expect(cacheGateAt).toBeGreaterThan(-1); expect(trayStopAt).toBeGreaterThan(-1); diff --git a/tests/update/update-stop-classification.test.ts b/tests/update/update-stop-classification.test.ts index 18057c50ba5..779ba4c4d0d 100644 --- a/tests/update/update-stop-classification.test.ts +++ b/tests/update/update-stop-classification.test.ts @@ -382,7 +382,8 @@ describe("stop failure classification (#3008)", () => { .toEqual({ proceed: false, reason: "proxy-unknown" }); // And no neighbouring status inherits the exemption. - for (const status of [1, 2, 4, 64, 78, 81, 130, null]) { + // 81 is the update handover code (#7423), pinned in update-teardown-handover.test.ts. + for (const status of [1, 2, 4, 64, 78, 82, 130, null]) { expect(decidePostStopUpdate({ status, teardownOutstanding: true, ...dead })) .toEqual({ proceed: false, reason: "stop-failed" }); } diff --git a/tests/update/update-teardown-handover.test.ts b/tests/update/update-teardown-handover.test.ts new file mode 100644 index 00000000000..c2ac7b59d58 --- /dev/null +++ b/tests/update/update-teardown-handover.test.ts @@ -0,0 +1,399 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CodexNativeRestoreResult } from "../../src/codex/inject"; +import { syncCodexOnStartIfEnabled } from "../../src/codex/desired-state"; +import { claimPendingTeardown, isUpdateHandoverReceipt, pendingTeardownPathFor, readPendingTeardown } from "../../src/config/pending-teardown"; +import { performStopTeardown } from "../../src/server/stop-teardown"; +import { runGuiUpdateWorker, updateJobPath } from "../../src/update/job"; +import { + STOP_HISTORY_DEFERRED_EXIT_CODE, + STOP_HISTORY_INCOMPLETE_EXIT_CODE, + STOP_TEARDOWN_HANDED_OVER_EXIT_CODE, +} from "../../src/update/stop-contract.mjs"; +import { decidePostStopUpdate } from "../../src/update/stop-decision.mjs"; +import { + dischargeUpdateTeardownHandovers, + forgetUpdateTeardownHandover, + stopHandsTeardownOver, + updateTeardownHandoverPending, + updateTeardownHandoverRequested, + UPDATE_TEARDOWN_HANDOVER_ENV, +} from "../../src/update/teardown-handover"; +import type { OcxConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { fixturePath, repoPath } from "../helpers/repo-root"; + +/** + * #7423: an update restart left a window where Codex could load the pristine catalog. + * + * The update's `ocx stop` restored native Codex, which removed every routed catalog row, + * and the new proxy appended them again only after it started. A Codex process that loaded + * the catalog in between treated routed models as unknown until it was restarted. These + * cases pin the handover that closes the window: the stop keeps its receipt instead of + * restoring, the proxy keeps its client config on exit, the updater proceeds past that one + * receipt, and the replacement releases it after its first successful publication. + */ + +const ENDPOINT = { hostname: "127.0.0.1", port: 10100 }; +const DEAD = () => false; +let home: string; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-teardown-handover-")); + process.env.OPENCODEX_HOME = home; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + removeTreeWithRetry(home); +}); + +const receiptNames = () => readdirSync(home).filter(name => name.startsWith("pending-teardown")); + +describe("handover exit code and update decision", () => { + test("81 is distinct from the history codes and from every literal code the CLI emits", () => { + expect(STOP_TEARDOWN_HANDED_OVER_EXIT_CODE).toBe(81); + expect([STOP_HISTORY_INCOMPLETE_EXIT_CODE, STOP_HISTORY_DEFERRED_EXIT_CODE]).not.toContain(81); + const cli = readFileSync(repoPath("src", "cli", "index.ts"), "utf8"); + const dispatch = readFileSync(repoPath("src", "cli", "dispatch.ts"), "utf8"); + const literal = [...cli.matchAll(/process\.exit(?:Code)?\s*(?:=|\()\s*(\d+)/g), ...dispatch.matchAll(/return (\d+);/g)] + .map(match => Number(match[1])); + expect(literal).not.toContain(81); + }); + + test("a handed-over stop proceeds past its own receipt and nothing else", () => { + const status = STOP_TEARDOWN_HANDED_OVER_EXIT_CODE; + expect(decidePostStopUpdate({ status, hasRuntimeState: false, liveness: "dead", teardownOutstanding: true })) + .toEqual({ proceed: true, reason: "handed-over" }); + expect(decidePostStopUpdate({ status, hasRuntimeState: true, liveness: "dead", teardownOutstanding: true }).reason).toBe("runtime-state"); + expect(decidePostStopUpdate({ status, hasRuntimeState: false, liveness: "live", teardownOutstanding: true }).reason).toBe("proxy-live"); + expect(decidePostStopUpdate({ status, hasRuntimeState: false, liveness: "unknown", teardownOutstanding: true }).reason).toBe("proxy-unknown"); + // An ordinary stop that leaves a receipt behind still aborts, exactly as before. + expect(decidePostStopUpdate({ status: 0, hasRuntimeState: false, liveness: "dead", teardownOutstanding: true }).reason) + .toBe("teardown-outstanding"); + }); +}); + +describe("stop handover predicate", () => { + const facts = (nonce: string | undefined) => ({ + requested: true, restoreBlocked: false, nonce, stopFailed: false, recoveredCount: 0, proxy: "stopped", deferralHonored: true, + }); + + test("hands over only this run's own receipt, alone, after a clean stop", () => { + const own = claimPendingTeardown(ENDPOINT, "exact", process.pid, "update-restart").nonce; + expect(stopHandsTeardownOver(facts(own))).toBe(true); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "stopped-orphan" })).toBe(true); + // A signal fallback ran the proxy's own exit cleanup, which already restored the pristine catalog. + expect(stopHandsTeardownOver({ ...facts(own), deferralHonored: false })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "stopped-orphan", deferralHonored: false })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), requested: false })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), restoreBlocked: true })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), stopFailed: true })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), recoveredCount: 1 })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "not-running" })).toBe(false); + expect(stopHandsTeardownOver(facts(undefined))).toBe(false); + }); + + test("a clean service-manager stop that took the proxy down may hand over", () => { + const own = claimPendingTeardown(ENDPOINT, "exact", process.pid, "update-restart").nonce; + expect(stopHandsTeardownOver({ ...facts(own), proxy: "not-running", service: "stopped" })).toBe(true); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "not-running", service: "stopped-respawnable" })).toBe(true); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "not-running", service: "absent" })).toBe(false); + expect(stopHandsTeardownOver({ ...facts(own), proxy: "not-running", service: "failed" })).toBe(false); + }); + + test("another obligation in the home forces the ordinary restore", () => { + const own = claimPendingTeardown(ENDPOINT, "exact", process.pid, "update-restart").nonce; + claimPendingTeardown(ENDPOINT, "exact", 999_999); + expect(stopHandsTeardownOver(facts(own))).toBe(false); + }); + + test("a guessed or unmarked receipt is never handed over", () => { + // A later ocx stop refuses to finish a guessed receipt, so handing one over could + // leave Codex routed at a dead proxy with only a manual way out. + const guessed = claimPendingTeardown(ENDPOINT, "guessed", process.pid, "update-restart").nonce; + expect(stopHandsTeardownOver(facts(guessed))).toBe(false); + rmSync(pendingTeardownPathFor(guessed), { force: true }); + const unmarked = claimPendingTeardown(ENDPOINT, "exact", process.pid).nonce; + expect(stopHandsTeardownOver(facts(unmarked))).toBe(false); + }); +}); + +describe("handover receipts", () => { + test("the handover marker round-trips and an unknown marker invalidates the receipt", () => { + const handover = claimPendingTeardown(ENDPOINT, "exact", 4242, "update-restart"); + const ordinary = claimPendingTeardown(ENDPOINT, "exact", 4242); + expect(isUpdateHandoverReceipt(handover.nonce)).toBe(true); + expect(isUpdateHandoverReceipt(ordinary.nonce)).toBe(false); + expect(isUpdateHandoverReceipt(null)).toBe(false); + const path = pendingTeardownPathFor(ordinary.nonce); + writeFileSync(path, JSON.stringify({ ...ordinary, handover: "anything" })); + expect(readPendingTeardown(ordinary.nonce).state).toBe("invalid"); + }); + + test("startup discharge releases only abandoned handover receipts", () => { + const abandoned = claimPendingTeardown(ENDPOINT, "exact", 4242, "update-restart").nonce; + const ordinary = claimPendingTeardown(ENDPOINT, "exact", 4243).nonce; + const running = claimPendingTeardown(ENDPOINT, "exact", 4244, "update-restart").nonce; + const result = dischargeUpdateTeardownHandovers(pid => pid === 4244); + expect(result).toEqual({ discharged: [abandoned], failed: [] }); + expect(existsSync(pendingTeardownPathFor(abandoned))).toBe(false); + expect(existsSync(pendingTeardownPathFor(ordinary))).toBe(true); + expect(existsSync(pendingTeardownPathFor(running))).toBe(true); + expect(dischargeUpdateTeardownHandovers(DEAD).discharged).toEqual([running]); + }); +}); + +describe("proxy side of the stop", () => { + const restored: CodexNativeRestoreResult = { success: true, message: "restored" } as CodexNativeRestoreResult; + + for (const handover of [true, false]) { + test(`a deferred ${handover ? "handover" : "ordinary"} stop ${handover ? "keeps" : "does not keep"} client config on exit`, async () => { + const nonce = claimPendingTeardown(ENDPOINT, "exact", 4242, handover ? "update-restart" : undefined).nonce; + let kept = 0; + let restores = 0; + const body = await performStopTeardown(new URL(`http://127.0.0.1:10100/api/stop?deferSharedTeardown=1&teardownNonce=${nonce}`), { + keepClientConfigOnExit: () => { kept += 1; }, + restoreNativeCodex: async () => { restores += 1; return restored; }, + stripGrok: () => ({ ok: true, changed: false, message: "" }), + }); + expect(body.sharedTeardown).toBe("deferred"); + expect(kept).toBe(handover ? 1 : 0); + expect(restores).toBe(0); + }); + } +}); + +describe("ocx stop under an update handover", () => { + const restore = { + success: true, message: "native Codex restored", + artifacts: { config: { state: "restored" }, catalog: { state: "restored" }, history: { state: "restored" } }, + }; + + function runStop(handover: boolean, extra: Record = {}) { + const child = spawnSync(process.execPath, [fixturePath("parent-stop-runner.ts")], { + cwd: repoPath(), + env: { ...process.env, OPENCODEX_HOME: home }, + input: JSON.stringify({ receipt: true, serverTeardown: true, json: true, response: null, restore, handover, ...extra }), + encoding: "utf8", + timeout: 20_000, + windowsHide: true, + }); + expect(child.error, child.stderr).toBeUndefined(); + const report = JSON.parse(readFileSync(join(home, "parent-stop-result.json"), "utf8")) as { + calls: { native: number; grok: number; cleared: number; service: number }; + nonce?: string; receiptExists: boolean; unexpectedIo: string[]; + }; + expect(report.unexpectedIo, child.stderr).toEqual([]); + return { ...report, exitCode: child.status, stdout: child.stdout, stderr: child.stderr }; + } + + test("keeps Codex routing and the catalog for the replacement and exits 81", () => { + const run = runStop(true); + expect(run.exitCode, run.stderr).toBe(STOP_TEARDOWN_HANDED_OVER_EXIT_CODE); + // Boundary one of the issue: nothing restored the pristine catalog after the old proxy stopped. + expect(run.calls).toMatchObject({ native: 0, grok: 0, cleared: 0 }); + expect(run.receiptExists).toBe(true); + expect(isUpdateHandoverReceipt(run.nonce ?? null)).toBe(true); + expect(JSON.parse(run.stdout)).toMatchObject({ proxy: "stopped", sharedTeardown: "skipped", exitCode: 81 }); + expect(decidePostStopUpdate({ status: run.exitCode, hasRuntimeState: false, liveness: "dead", teardownOutstanding: true })) + .toEqual({ proceed: true, reason: "handed-over" }); + }); + + test("an ordinary stop still restores native Codex and releases its receipt", () => { + const run = runStop(false); + expect(run.exitCode, run.stderr).toBe(0); + expect(run.calls).toMatchObject({ native: 1, grok: 1, cleared: 1 }); + expect(run.receiptExists).toBe(false); + expect(receiptNames()).toEqual([]); + }); + + test("hands over when the service manager stops the proxy itself", () => { + const run = runStop(true, { managerCascades: true, deadPidReadsNull: true }); + expect(run.exitCode, run.stderr).toBe(STOP_TEARDOWN_HANDED_OVER_EXIT_CODE); + expect(run.calls).toMatchObject({ native: 0, grok: 0, cleared: 0, service: 1 }); + expect(isUpdateHandoverReceipt(run.nonce ?? null)).toBe(true); + }); + + test("a manager-stopped ordinary stop still restores", () => { + const run = runStop(false, { managerCascades: true, deadPidReadsNull: true }); + expect(run.exitCode, run.stderr).toBe(0); + expect(run.calls).toMatchObject({ native: 1, grok: 1 }); + expect(receiptNames()).toEqual([]); + }); + + test("without a runtime record the stop restores instead of handing over a guessed receipt", () => { + const run = runStop(true, { noRuntimeRecord: true, nativeSignal: true }); + expect(run.exitCode, run.stderr).toBe(0); + expect(run.calls).toMatchObject({ native: 1, grok: 1 }); + expect(receiptNames()).toEqual([]); + }); + + test("a failed /api/stop that falls back to a signal restores instead of claiming a handover", () => { + // Exact runtime endpoint, but the proxy answers 503 before honoring the deferral, so the + // stop falls back to a signal and the proxy's own exit cleanup is what ran. + const run = runStop(true, { apiFailure: true, status: 503, response: { error: "unavailable" }, nativeSignal: true }); + expect(run.exitCode, run.stderr).toBe(0); + expect(run.calls).toMatchObject({ native: 1, grok: 1 }); + expect(receiptNames()).toEqual([]); + }); +}); + +describe("startup journal recovery under a handover", () => { + test("a pending handover is a valid, exact, abandoned handover receipt", () => { + expect(updateTeardownHandoverPending(DEAD)).toBe(false); + claimPendingTeardown(ENDPOINT, "exact", 4242); + expect(updateTeardownHandoverPending(DEAD)).toBe(false); + const guessed = claimPendingTeardown(ENDPOINT, "guessed", 4242, "update-restart").nonce; + expect(updateTeardownHandoverPending(DEAD)).toBe(false); + rmSync(pendingTeardownPathFor(guessed), { force: true }); + claimPendingTeardown(ENDPOINT, "exact", 4242, "update-restart"); + expect(updateTeardownHandoverPending(() => true)).toBe(false); + expect(updateTeardownHandoverPending(DEAD)).toBe(true); + }); + + test("the stop asks for a handover only when the replacement will run its startup sync", () => { + const env = { [UPDATE_TEARDOWN_HANDOVER_ENV]: "1" }; + expect(updateTeardownHandoverRequested(env, () => true)).toBe(true); + expect(updateTeardownHandoverRequested(env, () => false)).toBe(false); + expect(updateTeardownHandoverRequested(env, () => { throw new Error("unreadable config"); })).toBe(false); + expect(updateTeardownHandoverRequested({}, () => true)).toBe(false); + }); + + // The restarted proxy reconciles the previous proxy's journal before its first sync. Its + // owner PID is dead, so without adoption it restores native config for that interval. + for (const handover of [true, false]) { + test(`startup ${handover ? "adopts" : "restores"} a dead proxy's journal ${handover ? "with" : "without"} a pending handover`, () => { + const codexHome = mkdtempSync(join(tmpdir(), "ocx-handover-codex-")); + try { + const configPath = join(codexHome, "config.toml"); + const journalPath = join(codexHome, "opencodex-journal.json"); + const original = '# original\nmodel_provider = "openai"\n'; + const injected = '# injected\nmodel_provider = "opencodex"\n'; + writeFileSync(configPath, injected); + writeFileSync(journalPath, JSON.stringify({ + version: 1, + originalConfig: Buffer.from(original).toString("base64"), + originalProfile: null, + injectedConfigHash: createHash("sha256").update(injected).digest("hex"), + injectedProfileHash: null, + owner: { kind: "process", pid: 999_999 }, + pid: 999_999, + timestamp: new Date().toISOString(), + })); + if (handover) claimPendingTeardown(ENDPOINT, "exact", 999_998, "update-restart"); + const child = spawnSync(process.execPath, ["--eval", + 'const { reconcileStartupJournal } = require("./src/cli/cross-home-owner"); reconcileStartupJournal(); console.log(JSON.stringify({ pid: process.pid }));'], { + cwd: repoPath(), + env: { ...process.env, CODEX_HOME: codexHome, OPENCODEX_HOME: home }, + encoding: "utf8", + timeout: 60_000, + windowsHide: true, + }); + expect(child.status, child.stderr).toBe(0); + const childPid = (JSON.parse(child.stdout.trim().split("\n").at(-1)!) as { pid: number }).pid; + if (handover) { + expect(readFileSync(configPath, "utf8")).toBe(injected); + const journal = JSON.parse(readFileSync(journalPath, "utf8")) as { pid: number; owner: { pid: number }; originalConfig: string }; + expect(journal.pid).toBe(childPid); + expect(journal.owner.pid).toBe(childPid); + // The user's baseline survives the ownership change, so a later stop restores it. + expect(Buffer.from(journal.originalConfig, "base64").toString("utf8")).toBe(original); + } else { + expect(readFileSync(configPath, "utf8")).toBe(original); + } + } finally { + removeTreeWithRetry(codexHome); + } + }); + } + + function probe(mode: "swap" | "off"): Record { + const root = mkdtempSync(join(tmpdir(), `ocx-handover-probe-${mode}-`)); + try { + const child = spawnSync(process.execPath, [fixturePath("handover-journal-probe.ts"), mode, root], { + cwd: repoPath(), env: { ...process.env }, encoding: "utf8", timeout: 60_000, windowsHide: true, + }); + expect(child.status, child.stderr).toBe(0); + return JSON.parse(child.stdout.trim().split("\n").at(-1)!) as Record; + } finally { + removeTreeWithRetry(root); + } + } + + test("adoption never overwrites a Codex home replaced during recovery", () => { + expect(probe("swap")).toEqual({ replacementPreserved: true }); + }); + + test("a stale handover receipt does not keep routing once the Codex integration is off", () => { + expect(probe("off")).toEqual({ config: '# original\nmodel_provider = "openai"\n', journalExists: false }); + }); +}); + +describe("startup publication", () => { + const config = { providers: {} } as unknown as OcxConfig; + const cases: Array<[string, Record | undefined, boolean]> = [ + ["applied", { ok: true, status: "applied" }, true], + ["catalog-only", { ok: true, status: "catalog-only" }, true], + ["skipped", { ok: true, status: "skipped" }, false], + ["failed", { ok: false, status: "applied" }, false], + ["thrown", undefined, false], + ]; + for (const [name, outcome, published] of cases) { + test(`a ${name} startup sync ${published ? "publishes" : "does not publish"}`, async () => { + const result = await syncCodexOnStartIfEnabled(10100, config, async () => { + if (!outcome) throw new Error("sync failed"); + return outcome as never; + }); + expect(result.published).toBe(published); + }); + } +}); + +describe("dashboard update job", () => { + for (const restart of [true, false]) { + test(`${restart ? "requests" : "does not request"} the handover when restart=${restart}`, async () => { + let updateEnv: NodeJS.ProcessEnv | undefined; + // A worker launched by a recovery-started proxy could have inherited the marker. + process.env[UPDATE_TEARDOWN_HANDOVER_ENV] = "1"; + try { + await runGuiUpdateWorker(`handover-${restart}`, "latest", restart, { + checkForUpdateFn: () => ({ + currentVersion: "2.7.40", latestVersion: "2.7.41", channel: "latest", installer: "pnpm", + updateAvailable: true, canUpdate: true, command: "node /old/bin/ocx.mjs update --tag latest", + releaseNotesUrl: "https://github.com/lidge-jun/opencodex/releases/latest", + }), + resolvePnpmOwnerFn: () => ({ ok: true as const, owner: { + commandPath: "/pnpm/owner/bin/pnpm", packagePath: "/pnpm/owner/global/v11/node_modules/@bitkyc08/opencodex", + globalDir: "/pnpm/owner/global", globalRoot: "/pnpm/owner/global/v11", globalBinDir: "/pnpm/owner/bin", + } }), + resolvePnpmActiveLauncherFn: () => "/pnpm/owner/global/v11/node_modules/@bitkyc08/opencodex/bin/ocx.mjs", + integrityFn: () => ({ ok: true as const, integrity: "sha512-testfixturevalue000000000" }), + // Only the environment handed to the update child matters here; failing the command + // ends the job before any restart work, which other suites already cover. + runCommandFn: (_job, _bin, _args, _timeout, env) => { updateEnv = env; return { status: 1, signal: null }; }, + }); + } finally { + // The marker lives in this worker's shared environment; never let it reach a later test. + delete process.env[UPDATE_TEARDOWN_HANDOVER_ENV]; + rmSync(updateJobPath(), { force: true }); + } + expect(updateEnv).toBeDefined(); + expect(updateEnv?.[UPDATE_TEARDOWN_HANDOVER_ENV]).toBe(restart ? "1" : undefined); + }); + } + + test("a starting proxy forgets an inherited marker", () => { + const env: NodeJS.ProcessEnv = { [UPDATE_TEARDOWN_HANDOVER_ENV]: "1", KEEP: "x" }; + forgetUpdateTeardownHandover(env); + expect(env).toEqual({ KEEP: "x" }); + }); +}); +