From e4fcd5708baa2416c104d5b2b785ecd6a007225e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 04:27:29 +0000 Subject: [PATCH 1/2] Narrow MN-REQ-06.14 to a serve-only drain and restore. The upgrade stays inside memnet serve. Client retry and the systemd helper leave the requirement, the verify case, and the operator note. Co-authored-by: chouswei --- docs/README.md | 2 +- docs/operations/README.md | 2 +- docs/operations/safe-upgrade.md | 106 +++--------------- sysml-models/models/deploy.sysml | 21 +--- sysml-models/models/implementation.sysml | 31 +---- sysml-models/models/requirements.sysml | 57 +++++----- sysml-models/models/verify.sysml | 37 ++---- sysml-models/outputs/product-nest-one-page.md | 2 +- .../outputs/safe-upgrade-case-study.md | 11 +- .../outputs/ssot-to-code-allocate-map.md | 3 - tests/test_sysml_safe_upgrade.py | 23 ++-- 11 files changed, 85 insertions(+), 210 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0a3452e..5c9072a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -68,7 +68,7 @@ Multitask MUST for this product. Index: [`operations/README.md`](operations/READ | [`operations/product-gateway-contract.md`](operations/product-gateway-contract.md) | Product gateway on memnet-mcp (MN-REQ-06.12): route by product, house, or session | | [`operations/cluster-route-vs-slice-hand-carry.md`](operations/cluster-route-vs-slice-hand-carry.md) | Two named moves: ClusterRoute vs SliceHandCarry (#191 / #47 cousin; inventOnly) | | [`operations/admin-usage-report.md`](operations/admin-usage-report.md) | Admin-only serve usage JSON for a product-gate admin MCP (opaque alias; not agent MCP) | -| [`operations/safe-upgrade.md`](operations/safe-upgrade.md) | Drain and restore a live serve without dropping sessions (MN-REQ-06.14) | +| [`operations/safe-upgrade.md`](operations/safe-upgrade.md) | Drain, restart, and restore a live serve inside memnet serve (MN-REQ-06.14) | | [`operations/one-session-per-document.md`](operations/one-session-per-document.md) | One MemNet session per document over loopback serve (no MCP front) | Product skill: [`.cursor/skills/memnet-reference/`](../.cursor/skills/memnet-reference/). SysML trail: MN-REQ-12 → [`sysml-models/outputs/multitask-case-study.md`](../sysml-models/outputs/multitask-case-study.md). diff --git a/docs/operations/README.md b/docs/operations/README.md index b5ae5ea..bdafc5f 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -12,7 +12,7 @@ Agent operating doctrine for this product (not domain recipes). | [`product-gateway-contract.md`](product-gateway-contract.md) | Product gateway on memnet-mcp (MN-REQ-06.12): per-product route to one owning serve | | [`cluster-route-vs-slice-hand-carry.md`](cluster-route-vs-slice-hand-carry.md) | Two named moves: ClusterRoute vs SliceHandCarry (#191 / #47 cousin; inventOnly) | | [`admin-usage-report.md`](admin-usage-report.md) | Admin-only serve usage JSON (opaque alias; not agent MCP; MN-REQ-06.11) | -| [`safe-upgrade.md`](safe-upgrade.md) | Drain, restore, and `memnet-upgrade` so a serve swap keeps sessions (MN-REQ-06.14) | +| [`safe-upgrade.md`](safe-upgrade.md) | Drain and restore a live serve from inside memnet serve (MN-REQ-06.14) | | [`one-session-per-document.md`](one-session-per-document.md) | Product gate: one serve session per document over loopback (no MCP); 0.19.18 probe | Application pattern for `modelbasedPrj-*` / `SysMLEdgePrj-*`: [`../application-notes/system/llm-system-dev-multitask.md`](../application-notes/system/llm-system-dev-multitask.md). Index: [`../README.md`](../README.md). diff --git a/docs/operations/safe-upgrade.md b/docs/operations/safe-upgrade.md index 6b96c50..ef5a59d 100644 --- a/docs/operations/safe-upgrade.md +++ b/docs/operations/safe-upgrade.md @@ -1,112 +1,42 @@ # Safe serve upgrade -Upgrade `memnet-serve` without silently dropping a loaded session (MN-REQ-06.14). The agent loop does not change: cue, then `pin_map`, then `mutate`. This is a patch-level cut on 0.19. Do not bump the package version from this procedure. +Upgrade `memnet-serve` without dropping a loaded session (MN-REQ-06.14). The procedure stays inside the serve process: save every session, restart on the new version, reload every session. The agent loop does not change: cue, then `pin_map`, then `mutate`. -The manual procedure this replaces is a side-by-side virtualenv: install the new version, save every session, stop the old serve, start the new serve on the same port, reload, and check counts. Anything that failed to save was easy to miss. The drain below names every session it could not snapshot and refuses a ready-to-stop result unless you pass an explicit override. +During the restart, clients see `@ERR: serve_draining|retry_after_s=` or a connection error. There is no client retry window. -## Order +## Procedure -Deploy client tolerance **before** the serve swap. +1. Install the new version in a separate virtualenv. Leave the running venv in place. -1. Install the new `memnet-mcp` (and the droplet gateway, if this host is the gateway) so `serve_draining` and a brief connection refusal are retried. Leave the gateway's `pinned_version` on the version that is still running. -2. Confirm `MEMNET_UPGRADE_RETRY_S` is unset or about `30` on those client processes. `0` disables retry. -3. Only then run `memnet-upgrade` against the serve. - -During the gap, new `session_open` calls receive `@ERR: serve_draining|retry_after_s=` and clients back off. After the old process has exited and before the new one listens, connections are refused. That refusal is retried for the same window. A command the serve already accepted is not retried on timeout. - -Downtime that remains: a few seconds while the port is closed, covered by that retry. Sessions keep their ids, CapsPolicy ACL bindings, TTL expiry clock, and house (the session tag map). - -## What the serve does +```bash +python -m venv /opt/memnet/venv-new +/opt/memnet/venv-new/bin/pip install "memnet-llm==" +``` -Admin credential: `MEMNET_ADMIN_TOKEN` (same rule as the usage report). Unset is `@ERR: admin_unconfigured`. A mismatch is `@ERR: admin_denied`. This command is not an agent MCP tool. +2. Drain the running serve. `MEMNET_ADMIN_TOKEN` is required (unset is `@ERR: admin_unconfigured`; a mismatch is `@ERR: admin_denied`). This command is not an agent MCP tool. ```bash memnet admin upgrade-prepare --state-dir "$MEMNET_STATE_DIR" ``` -Envelope form: `{"upgrade_prepare": true, "admin_token": ""}`. +3. Check ready-to-stop. Stop the process only when stderr contains `@STAT: upgrade_prepare|ready|`. If a session cannot be snapshotted, the command names it, exits non-zero, and does not write a ready manifest. Pass `--allow-unsaved` only when those named sessions may be dropped. -Drain behaviour: +4. Point the unit `ExecStart` at the new venv, on the same port and the same `MEMNET_STATE_DIR`, and restart. -- Refuse new `session_open` with `serve_draining`. -- Finish commands already in flight, then refuse further commands. -- Snapshot every loaded session with the lossless snapshot writer. -- Write `upgrade-manifest.json` and `upgrade-snapshots/` under `MEMNET_STATE_DIR` (default `~/.local/state/memnet`). The manifest records session ids, row and edge counts, sha256 checksums, the serve version, and snapshot format `1`. -- An ACL and clock passport is an extra `# upgrade-passport` line. Older v1 loaders skip `#` lines, so a rollback can still read the graph. -- If any session raises `snapshot_unsaveable` (or any other save error), the command exits non-zero, lists that session in `upgrade-blocked.json`, and does **not** write a ready manifest. The serve keeps accepting work. Pass `--allow-unsaved` only when you accept dropping those named sessions. -- Do not stop the process unless stderr contains `@STAT: upgrade_prepare|ready|`. - -The new process reads the manifest on startup, reloads each snapshot, and checks counts and checksums. It prints: +5. Read the restore stat from the new process: ```text @STAT: upgrade_restore|ok||failed| ``` -Snapshot files stay on disk until `memnet admin upgrade-retire` after a clean restore report. A format this process does not support, or a checksum or parse failure, exits `3` and does not delete or rewrite the files. Restarting before retire replays the upgrade snapshots (writes made after the restore are not in those files). Retire as soon as the stat line is clean. - -## Helper - -`memnet-upgrade` encodes the side-by-side venv procedure. It refuses to start unless `--clients-ready` is set. - -```bash -memnet-upgrade \ - --clients-ready \ - --new-python /opt/memnet/venv-new/bin/python \ - --new-exec "/opt/memnet/venv-new/bin/memnet serve --host 127.0.0.1 --port 18765" \ - --state-dir /var/lib/memnet \ - --unit /etc/systemd/system/memnet-serve.service \ - --restart-unit memnet-serve -``` - -Steps: +A clean stat retires the manifest inside that process. A later restart does not replay the snapshots. The snapshot files stay on disk. A checksum failure, a parse failure, or an unsupported snapshot format exits `3`, leaves the files untouched, and does not retire. -1. Preflight: import `memnet` with the new interpreter and check `SUPPORTED_SNAPSHOT_FORMATS` against the current manifest (or format `1` when no manifest exists yet). -2. `admin upgrade-prepare` on the running serve. -3. Copy the unit to `memnet-serve.service.bak` and replace `ExecStart=`. -4. If `--gateway-config` and `--product` are set, copy the config to `*.bak` and set that product's `pinned_version` to the new version **after** the old serve is drained and **before** the new process listens. -5. Restart the unit. The new process restores and writes `upgrade-restore.json`. -6. If `failed` is not `0`, write the unit backup and the config backup back and restart again. - -`--state-dir` must be the same directory as the unit's `Environment=MEMNET_STATE_DIR`. The serve reads that variable at startup; the prepare command also receives `--state-dir`. - -Build the new venv beside the old one. Do not overwrite it until the restore has verified. - -```bash -python -m venv /opt/memnet/venv-new -/opt/memnet/venv-new/bin/pip install "memnet-llm==" -``` - -Use the index you trust. Do not put tokens or session ids in the unit file. Keep `MEMNET_ADMIN_TOKEN` in a root-only `EnvironmentFile`. - -## rpi5-syson - -Local engine on the Pi: - -| Process | Port | Unit (adjust to the host) | -|---------|------|---------------------------| -| `memnet serve` | `18765` | `memnet-serve.service` | -| `memnet-mcp` streamable-http | `18766` | `memnet-mcp-http.service` | - -Upgrade the MCP unit first (retry code, same serve version pin). Then `memnet-upgrade` the serve unit on `18765` with `MEMNET_STATE_DIR` on the Pi disk. Leave `18766` up so clients retry against the serve gap. - -## Endleaf engine serve - -The Endleaf product backend is the serve named in the droplet gateway registry (host and port live in that config, not in this repo). Use that unit's port and state directory. The drain and restore are the same command. Do not point the gateway at a second backend to "move" live sessions. - -## Droplet gateway - -The gateway process is `memnet-mcp --transport gateway` with `MEMNET_GATEWAY_CONFIG`. Ship the retry-capable build first, with `pinned_version` still equal to the running engine. Then, in the same `memnet-upgrade` invocation that swaps the engine, pass: - -```bash ---gateway-config /etc/memnet/gateway.json \ ---product endleaf \ ---restart-unit memnet-gateway -``` +## Rollback -The pin changes only after drain succeeds, and it is restored from `gateway.json.bak` if the engine restore fails. Restart the gateway after the pin write so it stops caching the previous version (`version_cache_s`). No plaintext credentials in git. Hashes stay in the config file on the droplet. +Point the unit `ExecStart` back at the old venv and start it when the restore stat is not clean. After a clean stat the manifest is retired, so a later start does not reload those snapshots. -## Rollback +## Hosts -A failed verification restores the previous `ExecStart=` and the previous pin, then restarts. Snapshot files are still in the state directory. The old venv loads format v1 snapshots (`#` passport lines are ignored). That reload keeps the session id and the graph. The old loader rebases the TTL clock from `ttl_minutes` and does not read ACL bindings from the passport; re-grant those on the old serve if you stay there. The new serve's restore path keeps the id, the ACL bindings, the expiry instant, and the house. +On rpi5-syson the serve listens on `18765` (`memnet-serve.service`). `memnet-mcp` on `18766` is not part of this procedure. The Endleaf engine serve is the backend named in the gateway registry; use that unit's port and state directory. The droplet gateway is unchanged: do not edit its version pin as part of this restart. -Neighbourhood reserves are not part of the snapshot. Take a fresh reserve after the new serve is up. +Do not put tokens or session ids in the unit file. Keep `MEMNET_ADMIN_TOKEN` in a root-only `EnvironmentFile`. diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index 757daf6..fa5b234 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -1323,11 +1323,12 @@ package MemNet { part def SafeServeUpgrade { doc /* - Drain, manifest, and startup restore for a live memnet-serve + Drain, manifest, and startup restore inside memnet serve (MN-REQ-06.14). Lossless snapshot path. No silent session loss. Older patch snapshot format v1 loads. Corrupt restore - fails loud and leaves snapshot files in place. Client retry - is on memnet-mcp and the product gateway, deployed first. + fails loud and leaves snapshot files in place. A clean + restore retires the manifest in that same startup. No + client retry. No systemd or gateway-pin helper. */ attribute implemented : Boolean = true; attribute noSemVerBump : Boolean = true; @@ -1351,9 +1352,7 @@ package MemNet { attribute preservesAcl : Boolean = true; attribute preservesTtlClock : Boolean = true; attribute preservesHouse : Boolean = true; - attribute retryDefaultS : Integer = 30; - attribute clientsBeforeServe : Boolean = true; - attribute helperRollsBack : Boolean = true; + attribute retiresAfterCleanRestore : Boolean = true; attribute statUpgradeRestore : String = "upgrade_restore"; satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_14_SafeServeUpgrade; @@ -1836,10 +1835,6 @@ package MemNet { attribute noRequiredIdTools : Boolean = true; attribute adminUsageToolNested : Boolean = false; attribute productLabelOnSessionOpen : Boolean = false; - attribute upgradeRetry : Boolean = true; - attribute upgradeRetryDefaultS : Integer = 30; - attribute retryServeDraining : Boolean = true; - attribute retryConnectionRefusal : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; @@ -1864,7 +1859,6 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_04_SliceEconomy::MN_REQ_04_9_SessionOutlineEmptyCue; satisfy MN_REQ_00_MissionBridge::MN_REQ_11_SnapshotInterop::MN_REQ_11_17_CatalogModelSnap; satisfy MN_REQ_00_MissionBridge::MN_REQ_11_SnapshotInterop::MN_REQ_11_17_CatalogModelSnap::MN_REQ_11_17_1_CrossCutSatisfyLocators; - satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_14_SafeServeUpgrade; } part def MemNetCoreLibrary { @@ -3641,13 +3635,8 @@ package MemNet { attribute adminCountsOmitSessionId : Boolean = true; attribute singleBackendUnchanged : Boolean = true; attribute hardcodedHost : Boolean = false; - attribute upgradeRetry : Boolean = true; - attribute upgradeRetryDefaultS : Integer = 30; - attribute retryServeDraining : Boolean = true; - attribute retryConnectionRefusal : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_12_ProductGateway; - satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_14_SafeServeUpgrade; satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_9_LanMcpFrontSeveralServes; } diff --git a/sysml-models/models/implementation.sysml b/sysml-models/models/implementation.sysml index b17c9dd..534d1b0 100644 --- a/sysml-models/models/implementation.sysml +++ b/sysml-models/models/implementation.sysml @@ -123,20 +123,10 @@ package MemNetImplementation { attribute pyName : String = "memnet.admin_usage"; } part def SafeUpgradeMod { - doc /* Serve drain, manifest, and startup restore (MN-REQ-06.14). */ + doc /* Serve drain, manifest, startup restore, and retire (MN-REQ-06.14). */ attribute path : String = "parts/common/memnet/memnet/upgrade.py"; attribute pyName : String = "memnet.upgrade"; } - part def UpgradeRetryMod { - doc /* Bounded retry for serve_draining and a brief connection refusal. */ - attribute path : String = "parts/common/memnet/memnet/upgrade_retry.py"; - attribute pyName : String = "memnet.upgrade_retry"; - } - part def UpgradeHelperMod { - doc /* Side-by-side venv helper: preflight, drain, swap, verify, rollback. */ - attribute path : String = "parts/common/memnet/memnet/upgrade_run.py"; - attribute pyName : String = "memnet.upgrade_run"; - } part def CliFacadeMod { attribute path : String = "parts/common/memnet/memnet/cli.py"; attribute pyName : String = "memnet.cli"; @@ -289,8 +279,6 @@ package MemNetImplementation { part serveDaemonMod : ServeDaemonMod; part adminUsageMod : AdminUsageReportMod; part safeUpgradeMod : SafeUpgradeMod; - part upgradeRetryMod : UpgradeRetryMod; - part upgradeHelperMod : UpgradeHelperMod; part cliMod : CliFacadeMod; part sessionsMod : SessionLifecycleMod; part storeMod : GraphStoreMod; @@ -373,25 +361,10 @@ package MemNetImplementation { end code ::> adminUsageMod; } allocation safeUpgradeToMod : SoftwareAllocate { - doc /* Drain, manifest, and startup restore on memnet-serve. */ + doc /* Drain, manifest, startup restore, and retire on memnet-serve. */ end logical ::> memNetSystem.core.transport.tcp.safeUpgrade; end code ::> safeUpgradeMod; } - allocation upgradeRetryToMod : SoftwareAllocate { - doc /* Gateway retry across serve_draining and a brief refusal. */ - end logical ::> lanMcpFront.productGateway; - end code ::> upgradeRetryMod; - } - allocation mcpUpgradeRetryToMod : SoftwareAllocate { - doc /* memnet-mcp TCP client retry; same helper as the gateway. */ - end logical ::> memNetSystem.mcp.bridge; - end code ::> upgradeRetryMod; - } - allocation upgradeHelperToMod : SoftwareAllocate { - doc /* Operator helper memnet-upgrade. Not an agent MCP tool. */ - end logical ::> memNetSystem.core.cli; - end code ::> upgradeHelperMod; - } allocation cliToMod : SoftwareAllocate { end logical ::> memNetSystem.core.cli; end code ::> cliMod; diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index 2532c4e..8d10367 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -36,7 +36,7 @@ MN-REQ-06.11 Admin-only serve usage report (counts/caps; opaque session alias; not agent MCP) MN-REQ-06.12 Product gateway on memnet-mcp: per-product route to owning serve (not mn_tip_) MN-REQ-06.13 Serve request isolation: each command's stdout, stderr, and exit_code - MN-REQ-06.14 Safe serve upgrade: drain, manifest, restore; no silent session loss + MN-REQ-06.14 Safe serve upgrade inside memnet serve: drain, manifest, restore, retire; no silent session loss MN-REQ-11.17 Catalog Snap / model Snap — session strata (0.15) MN-REQ-11.17.1 Cross-cut satisfy locators on the catalog; SysMLEdge pin_map is a different tool (not this catalog) @@ -1139,18 +1139,24 @@ package MemNetRequirements { requirement def MN_REQ_06_14_SafeServeUpgrade { doc /* - SHALL upgrade a live memnet-serve (and the memnet-mcp / - product gateway in front of it) without silently dropping - a loaded session. Same usage method (cue then pin_map, - mutate). No SemVer bump. Patch-level c on 0.19. + SHALL upgrade a live memnet-serve without silently + dropping a loaded session. The procedure stays inside + memnet serve: snapshot every loaded session, restart + on the new version, reload every session. Same usage + method (cue then pin_map, mutate). No SemVer bump. + Patch-level c on 0.19. memnet-mcp and the product + gateway are outside this procedure. During the restart + they see serve_draining or a connection error. There + is no client retry window and no helper that edits a + systemd unit or a gateway version pin. Admin drain: credential MEMNET_ADMIN_TOKEN (same gate as MN-REQ-06.11; unset admin_unconfigured; mismatch admin_denied). Command `memnet admin upgrade-prepare` and envelope {upgrade_prepare:true, admin_token}. Not an agent MCP tool (onAgentMcp=false). - Drain refuses a new session_open with - @ERR: serve_draining|retry_after_s= (retryable). - It finishes commands already in flight, then snapshots + Drain enters quiesce. It refuses a new command with + @ERR: serve_draining|retry_after_s=. It + finishes commands already in flight, then snapshots every loaded session through the lossless snapshot path (MN-REQ-01.9). It writes a manifest in the state dir (MEMNET_STATE_DIR): session ids, row and edge counts, @@ -1163,31 +1169,22 @@ package MemNetRequirements { New serve startup reads that manifest, reloads every saved session, and checks counts and checksums. It reports @STAT: upgrade_restore|ok|n|failed|m. It keeps - the manifest and snapshot files until that restore is - verified. A snapshot written by an older patch (format - v1) SHALL load. If the format is unsupported or a - snapshot is corrupt, startup SHALL fail loudly and - SHALL NOT delete or rewrite the snapshot files, so a - rollback to the old venv can still reload them. + the snapshot files. A snapshot written by an older patch + (format v1) SHALL load. If the format is unsupported or + a snapshot is corrupt, startup SHALL fail loudly and + SHALL NOT delete or rewrite the snapshot files. + A clean restore (failed = 0) SHALL retire the manifest + inside that same startup, before the process accepts + work. The operator does not run a separate retire + command. A later restart SHALL NOT replay those + snapshots. A failed restore SHALL NOT retire. Restored sessions keep the same session id, CapsPolicy ACL bindings, TTL expiry clock, and house (the session tag map). Product label is kept with the passport. - memnet-mcp (TCP) and the product gateway SHALL treat - serve_draining and a brief connection refusal as - retryable with bounded backoff for MEMNET_UPGRADE_RETRY_S - (default 30 seconds). Order: deploy that client - tolerance first, then drain and swap the serve. The - gateway backend version pin is updated in the same - procedure, after the old serve has stopped and before - the new serve listens. - Helper `memnet-upgrade` encodes the side-by-side venv - procedure: preflight the new interpreter against the - snapshot format, drain, swap the systemd ExecStart from - a unit backup, restore, verify, and roll back to the - backup unit and the previous pin if verification fails. - Hosts: rpi5-syson (serve 18765, mcp 18766), the Endleaf - engine serve named by the gateway registry, and the - droplet gateway. No real session id in docs or tests. + Operator: install the new venv, run upgrade-prepare, + check ready_to_stop, restart the unit on the new venv, + read the restore stat. To roll back, point the unit at + the old venv. No real session id in docs or tests. */ attribute requirementId : String = "MN-REQ-06.14"; } diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 225cb2d..fb110f7 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -52,7 +52,7 @@ catalogue stays inventOnly. Safe serve upgrade (MN-VER-06-S12): SafeServeUpgrade on TcpServeBridge; admin drain; manifest; startup restore; - client retry on MCP and the product gateway; no silent + retire after a clean restore; no client retry; no silent session loss; no SemVer bump. Optional id nickname TARGET (MN-VER-02-S01): GraphElement identity; CREATE () legal; HiddenStoreHandle off-wire; 0.9 leftover_by_id store. @@ -1632,30 +1632,25 @@ package MemNetVerification { and gateway.adminCountsOmitSessionId == true and gateway.singleBackendUnchanged == true and gateway.hardcodedHost == false - and gateway.upgradeRetry == true - and gateway.retryServeDraining == true - and gateway.retryConnectionRefusal == true } } } verification def MN_VER_06_S12_SafeServeUpgrade { doc /* - MN-REQ-06.14 — drain, manifest, and startup restore. - Admin credential. New session_open refuses - serve_draining. Unsaveable sessions block ready_to_stop - unless allow-unsaved. Restore checks counts and - checksums. Corrupt snapshots stay on disk. Same session - id, ACL, TTL clock, and house. MCP and gateway retry - (default 30s) ship before the serve swap. Helper rolls - back. Not an agent MCP tool. No SemVer bump. + MN-REQ-06.14 — drain, manifest, and startup restore + inside memnet serve. Admin credential. Quiesce refuses + new commands with serve_draining. Unsaveable sessions + block ready_to_stop unless allow-unsaved. Restore checks + counts and checksums. Corrupt snapshots stay on disk. + A clean restore retires the manifest in that startup. + Same session id, ACL, TTL clock, and house. Not an + agent MCP tool. No client retry. No SemVer bump. */ attribute verificationId : String = "MN-VER-06-S12"; subject tcp : TcpServeBridge; - subject mcp : McpFacade; subject cli : CliFacade; - subject gateway : MemNetProductGateway; objective safeServeUpgrade { verify safeServeUpgradeReq; @@ -1682,20 +1677,10 @@ package MemNetVerification { and tcp.safeUpgrade.preservesAcl == true and tcp.safeUpgrade.preservesTtlClock == true and tcp.safeUpgrade.preservesHouse == true - and tcp.safeUpgrade.retryDefaultS == 30 - and tcp.safeUpgrade.clientsBeforeServe == true - and tcp.safeUpgrade.helperRollsBack == true - and mcp.upgradeRetry == true - and mcp.upgradeRetryDefaultS == 30 - and mcp.retryServeDraining == true - and mcp.retryConnectionRefusal == true - and mcp.adminUsageToolNested == false + and tcp.safeUpgrade.retiresAfterCleanRestore == true + and tcp.safeUpgrade.statUpgradeRestore == "upgrade_restore" and cli.upgradePrepareOnAgentMcp == false and cli.upgradePrepare.onAgentMcp == false - and gateway.upgradeRetry == true - and gateway.upgradeRetryDefaultS == 30 - and gateway.retryServeDraining == true - and gateway.retryConnectionRefusal == true } } } diff --git a/sysml-models/outputs/product-nest-one-page.md b/sysml-models/outputs/product-nest-one-page.md index 9a74dd0..f13e79a 100644 --- a/sysml-models/outputs/product-nest-one-page.md +++ b/sysml-models/outputs/product-nest-one-page.md @@ -26,7 +26,7 @@ ARCHIVE LOOK MemNetArchive (models/archive.sysml) — leftover_* / ProjectMemNet MUST NOT import OPS LOOK MemNetUsageDashboard — human look only; not agent wire SERVE ADMIN LOOK AdminUsageReport on TcpServeBridge — admin JSON; opaque alias; not agent MCP (MN-REQ-06.11) -SERVE UPGRADE SafeServeUpgrade on TcpServeBridge — drain, manifest, restore (MN-REQ-06.14) +SERVE UPGRADE SafeServeUpgrade on TcpServeBridge — drain, manifest, restore, retire (MN-REQ-06.14) OPS FLEET MemNetOpsFleet — device MemNet services; one MemNet MCP at droplet (tip/ops; tip≠face; product face is sysmledge) OPS ACCESS TipMemNetAccessPortal — Szu-Wei invite, Google login, Bearer for keyed tip MCP at droplet WWW (tip≠face; not sysmledge; portal sidecar; look-only service/client status) OPS LAN FRONT MemNetLanMcpFront — ClusterRoute: one MCP catalogue over N LAN serves (inventOnly #191; tip≠face; not shipped) diff --git a/sysml-models/outputs/safe-upgrade-case-study.md b/sysml-models/outputs/safe-upgrade-case-study.md index 841ea92..b916ae9 100644 --- a/sysml-models/outputs/safe-upgrade-case-study.md +++ b/sysml-models/outputs/safe-upgrade-case-study.md @@ -1,6 +1,6 @@ # Case study: safe serve upgrade (MN-REQ-06.14) -**Story:** A live `memnet-serve` has to move to a new patch without dropping sessions on the floor. The hand procedure (side-by-side venv, save, stop, start, reload) worked, and it also hid any session that failed to save. +**Story:** A live `memnet-serve` has to move to a new patch without dropping sessions. The hand procedure (side-by-side venv, save, stop, start, reload) worked, and it also hid any session that failed to save. The shipped cut keeps that procedure inside the serve process. **Requirement:** MN-REQ-06.14 (`safeServeUpgradeReq`). **Verify:** MN-VER-06-S12. @@ -8,13 +8,10 @@ |--|--| | Part | `SafeServeUpgrade` on `TcpServeBridge` | | Command | `CmdAdminUpgradePrepare` on `CliFacade`; not on `McpFacade` | -| Retry | `McpFacade` and `MemNetProductGateway` (`upgradeRetry=true`, default 30s) | -| Helper | `memnet-upgrade` (`UpgradeHelperMod`) | | Snapshot | lossless writer plus an optional `# upgrade-passport` line | +| Retire | automatic after a clean restore, inside that startup | | Flag | `implemented=true`; `noSemVerBump=true`; `silentSessionLoss=false` | -Drain refuses new `session_open` with `serve_draining`. It snapshots through the lossless path, writes a manifest (counts, checksums, format version), and stays not-ready when any session is `snapshot_unsaveable` unless `--allow-unsaved` is set. Startup restore checks counts and checksums and prints `@STAT: upgrade_restore|ok|n|failed|m`. A corrupt file or an unsupported format fails loud and leaves the files in place. +Drain enters quiesce and refuses new commands with `serve_draining`. It finishes in-flight work, snapshots through the lossless path, and writes a manifest (counts, checksums, format version). It stays not-ready when any session is `snapshot_unsaveable` unless `--allow-unsaved` is set. Startup restore checks counts and checksums and prints `@STAT: upgrade_restore|ok|n|failed|m`. A clean restore retires the manifest before the process accepts work, so a later restart does not replay those snapshots. A corrupt file or an unsupported format fails loud, leaves the files in place, and does not retire. -Client tolerance ships first. Then the serve swaps. The gateway pin moves in that same procedure, after drain and before the new process listens. - -Hosts in the operator note: rpi5-syson serve `18765` and mcp `18766`; the Endleaf engine serve named by the gateway registry; the droplet gateway. Teach: [`docs/operations/safe-upgrade.md`](../../docs/operations/safe-upgrade.md). +memnet-mcp and the product gateway are not part of the procedure. Clients see the refusal or a connection error while the port is down. Teach: [`docs/operations/safe-upgrade.md`](../../docs/operations/safe-upgrade.md). diff --git a/sysml-models/outputs/ssot-to-code-allocate-map.md b/sysml-models/outputs/ssot-to-code-allocate-map.md index 714e989..6150d78 100644 --- a/sysml-models/outputs/ssot-to-code-allocate-map.md +++ b/sysml-models/outputs/ssot-to-code-allocate-map.md @@ -20,9 +20,6 @@ | tcpDaemonToMod | `…transport.tcp` | `serveDaemonMod` | `parts/common/memnet/memnet/serve.py` | `memnet.serve` | true | | adminUsageToMod | `…transport.tcp.adminUsage` | `adminUsageMod` | `parts/common/memnet/memnet/admin_usage.py` | `memnet.admin_usage` | true | | safeUpgradeToMod | `…transport.tcp.safeUpgrade` | `safeUpgradeMod` | `parts/common/memnet/memnet/upgrade.py` | `memnet.upgrade` | true | -| upgradeRetryToMod | `lanMcpFront.productGateway` | `upgradeRetryMod` | `parts/common/memnet/memnet/upgrade_retry.py` | `memnet.upgrade_retry` | true | -| mcpUpgradeRetryToMod | `memNetSystem.mcp.bridge` | `upgradeRetryMod` | `parts/common/memnet/memnet/upgrade_retry.py` | `memnet.upgrade_retry` | true | -| upgradeHelperToMod | `memNetSystem.core.cli` | `upgradeHelperMod` | `parts/common/memnet/memnet/upgrade_run.py` | `memnet.upgrade_run` | true | | cliToMod | `memNetSystem.core.cli` | `cliMod` | `parts/common/memnet/memnet/cli.py` | `memnet.cli` | true | | sessionsToMod | `…sessions` | `sessionsMod` | `parts/common/memnet/memnet/session_lifecycle.py` | `memnet.session_lifecycle` | true | | storeToMod | `…sessions.store` | `storeMod` | `parts/common/memnet/memnet/graph_store.py` | `memnet.graph_store` | true | diff --git a/tests/test_sysml_safe_upgrade.py b/tests/test_sysml_safe_upgrade.py index b5bc7fa..d87c368 100644 --- a/tests/test_sysml_safe_upgrade.py +++ b/tests/test_sysml_safe_upgrade.py @@ -23,13 +23,16 @@ def test_safe_upgrade_parts(): assert "part safeUpgrade : SafeServeUpgrade" in text assert "attribute silentSessionLoss : Boolean = false" in text assert 'attribute errDraining : String = "serve_draining"' in text - assert "attribute retryDefaultS : Integer = 30" in text - assert "attribute clientsBeforeServe : Boolean = true" in text + assert "attribute retiresAfterCleanRestore : Boolean = true" in text + assert "attribute retryDefaultS" not in text + assert "attribute clientsBeforeServe" not in text + assert "attribute helperRollsBack" not in text mcp = text.split("part def McpFacade", 1)[1].split("part def ", 1)[0] assert "part upgradePrepare" not in mcp - assert "attribute upgradeRetry : Boolean = true" in mcp + assert "upgradeRetry" not in mcp gateway = text.split("part def MemNetProductGateway", 1)[1].split("part def ", 1)[0] - assert "attribute retryConnectionRefusal : Boolean = true" in gateway + assert "upgradeRetry" not in gateway + assert "MN_REQ_06_14_SafeServeUpgrade" not in gateway def test_requirement_verify_allocate(): @@ -47,11 +50,13 @@ def test_requirement_verify_allocate(): assert "part def SafeUpgradeMod" in impl assert "allocation safeUpgradeToMod" in impl assert "memnet.upgrade" in impl - assert "memnet.upgrade_retry" in impl - assert "memnet.upgrade_run" in impl + assert "memnet.upgrade_retry" not in impl + assert "memnet.upgrade_run" not in impl + assert "retiresAfterCleanRestore == true" in ver ledger = MAP.read_text(encoding="utf-8") assert "| safeUpgradeToMod |" in ledger - assert "| upgradeHelperToMod |" in ledger + assert "upgradeHelperToMod" not in ledger + assert "upgradeRetryToMod" not in ledger def test_outputs_and_docs(): @@ -61,7 +66,9 @@ def test_outputs_and_docs(): nest = NEST.read_text(encoding="utf-8") assert "SafeServeUpgrade" in nest teach = TEACH.read_text(encoding="utf-8") - assert "memnet-upgrade" in teach + assert "memnet-upgrade" not in teach + assert "MEMNET_UPGRADE_RETRY_S" not in teach + assert "upgrade-prepare" in teach assert "MEMNET_ADMIN_TOKEN" in teach assert "18765" in teach assert "18766" in teach From 31356411ab5b9ef8ca5bff88d9f131e49f06e65f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 04:27:29 +0000 Subject: [PATCH 2/2] Drop the upgrade retry helper and retire a clean restore inside serve. memnet-mcp and the gateway no longer retry serve_draining. A clean startup restore marks the manifest retired so a later restart does not replay the snapshots. Co-authored-by: chouswei --- parts/common/memnet/memnet/cli.py | 24 -- parts/common/memnet/memnet/serve.py | 4 - parts/common/memnet/memnet/upgrade.py | 28 +- parts/common/memnet/memnet/upgrade_retry.py | 67 ----- parts/common/memnet/memnet/upgrade_run.py | 258 ------------------ .../memnet-mcp/software/memnet_mcp/client.py | 16 +- .../software/memnet_mcp/product_gateway.py | 44 ++- .../software/memnet_mcp/serve_bridge.py | 9 +- pyproject.toml | 1 - tests/conftest.py | 6 - tests/test_safe_upgrade.py | 194 ++++--------- 11 files changed, 76 insertions(+), 575 deletions(-) delete mode 100644 parts/common/memnet/memnet/upgrade_retry.py delete mode 100644 parts/common/memnet/memnet/upgrade_run.py diff --git a/parts/common/memnet/memnet/cli.py b/parts/common/memnet/memnet/cli.py index 8652aab..f53be7c 100644 --- a/parts/common/memnet/memnet/cli.py +++ b/parts/common/memnet/memnet/cli.py @@ -569,30 +569,6 @@ def admin_upgrade_prepare( raise typer.Exit(result.exit_code) -@admin_app.command("upgrade-retire") -def admin_upgrade_retire( - token: Annotated[ - str | None, - typer.Option("--token", help="Admin token presented by the caller (do not log)."), - ] = None, - state_dir: Annotated[str | None, typer.Option("--state-dir")] = None, -) -> None: - """Stop replaying an upgrade manifest after a verified restore. Keeps the files.""" - from pathlib import Path - - from memnet.admin_usage import authenticate, caller_token - from memnet.upgrade import retire_manifest - - presented = token if token else caller_token() - try: - authenticate(presented) - retire_manifest(Path(state_dir) if state_dir else None) - except MemNetError as exc: - _handle_error(exc) - emit_stdout('{"ok": true, "retired": true}') - emit_stderr("@STAT: upgrade_retire|1|-") - - @session_app.command("expire-status") def session_expire_status() -> None: """Booleans for expire-save (serve_status). Path redacted; no sids.""" diff --git a/parts/common/memnet/memnet/serve.py b/parts/common/memnet/memnet/serve.py index 48582f1..09e1c46 100644 --- a/parts/common/memnet/memnet/serve.py +++ b/parts/common/memnet/memnet/serve.py @@ -92,10 +92,6 @@ def _handle_request(payload: dict[str, Any]) -> dict[str, Any]: from memnet.upgrade import upgrade_prepare_envelope return upgrade_prepare_envelope(token_s, allow_unsaved=bool(payload.get("allow_unsaved"))) - if payload.get("upgrade_retire") is True: - from memnet.upgrade import upgrade_retire_envelope - - return upgrade_retire_envelope(token_s) argv = payload.get("args", []) if not isinstance(argv, list): diff --git a/parts/common/memnet/memnet/upgrade.py b/parts/common/memnet/memnet/upgrade.py index 3f54ac9..87ffb8e 100644 --- a/parts/common/memnet/memnet/upgrade.py +++ b/parts/common/memnet/memnet/upgrade.py @@ -1,8 +1,9 @@ """Safe serve upgrade: drain, manifest, and startup restore (MN-REQ-06.14). Admin-only. Not an agent MCP tool. A session that cannot be snapshotted -blocks ready-to-stop unless the operator passes allow-unsaved. Snapshot -files are not deleted when restore fails. +blocks ready-to-stop unless the operator passes allow-unsaved. A clean +restore retires the manifest in that same startup. Snapshot files are +not deleted, and a failed restore does not retire. """ from __future__ import annotations @@ -80,10 +81,7 @@ def _drain_wait_s() -> float: def _is_upgrade_argv(argv: list[Any]) -> bool: if len(argv) < 2: return False - return str(argv[0]) == "admin" and str(argv[1]) in { - "upgrade-prepare", - "upgrade-retire", - } + return str(argv[0]) == "admin" and str(argv[1]) == "upgrade-prepare" class DrainGate: @@ -581,6 +579,7 @@ def startup_restore_or_exit(directory: Path | None = None) -> RestoreReport: sys.stderr.write(line) if report.failed: raise SystemExit(3) + retire_manifest(directory) return report @@ -627,20 +626,3 @@ def upgrade_prepare_envelope( else: stderr = result.stat + "\n" return {"exit_code": result.exit_code, "stdout": result.stdout_json(), "stderr": stderr} - - -def upgrade_retire_envelope(token: str | None) -> dict[str, Any]: - try: - authenticate(token) - retire_manifest() - except MemNetError as exc: - return { - "exit_code": exc.exit_code, - "stdout": "", - "stderr": f"@ERR: {exc.code}|{exc.message}\n", - } - return { - "exit_code": 0, - "stdout": json.dumps({"ok": True, "retired": True}) + "\n", - "stderr": "@STAT: upgrade_retire|1|-\n", - } diff --git a/parts/common/memnet/memnet/upgrade_retry.py b/parts/common/memnet/memnet/upgrade_retry.py deleted file mode 100644 index 36dd5e4..0000000 --- a/parts/common/memnet/memnet/upgrade_retry.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Bounded retry while a serve is draining or briefly refusing connections. - -Default window is 30 seconds (``MEMNET_UPGRADE_RETRY_S``). A window of 0 -tries once. Timeout is not retried: the serve may already have accepted -the command. ``serve_draining`` is retried because the serve refused it. -""" - -from __future__ import annotations - -import os -import re -import time -from collections.abc import Callable -from typing import Any - -ENV_RETRY_S = "MEMNET_UPGRADE_RETRY_S" -DEFAULT_RETRY_S = 30.0 -_RETRY_AFTER = re.compile(r"retry_after_s=(\d+(?:\.\d+)?)") -_RETRY_EXC = (ConnectionRefusedError, ConnectionResetError, ConnectionAbortedError, BrokenPipeError) - - -def retry_window_s() -> float: - raw = os.environ.get(ENV_RETRY_S) - if raw is None or raw.strip() == "": - return DEFAULT_RETRY_S - try: - return max(0.0, float(raw)) - except ValueError: - return DEFAULT_RETRY_S - - -def _retry_after(stderr: str) -> float: - match = _RETRY_AFTER.search(stderr or "") - if not match: - return 0.05 - return max(0.0, float(match.group(1))) - - -def call_with_upgrade_retry(fn: Callable[[], dict[str, Any]]) -> dict[str, Any]: - """Call ``fn`` until it returns or the retry window is spent. - - ``fn`` raises a connection-refusal error, or returns an envelope whose - stderr contains ``serve_draining``. - """ - window = retry_window_s() - deadline = time.monotonic() + window - delay = 0.05 - while True: - try: - raw = fn() - except _RETRY_EXC: - if window <= 0 or time.monotonic() >= deadline: - raise - remaining = deadline - time.monotonic() - time.sleep(min(delay, remaining)) - delay = min(delay * 2, 2.0) - continue - stderr = "" - if isinstance(raw, dict): - stderr = str(raw.get("stderr") or "") - if "serve_draining" in stderr and window > 0 and time.monotonic() < deadline: - remaining = deadline - time.monotonic() - pause = min(_retry_after(stderr), delay, remaining) - time.sleep(pause) - delay = min(delay * 2, 2.0) - continue - return raw diff --git a/parts/common/memnet/memnet/upgrade_run.py b/parts/common/memnet/memnet/upgrade_run.py deleted file mode 100644 index 4548edb..0000000 --- a/parts/common/memnet/memnet/upgrade_run.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Operator helper for a side-by-side venv upgrade (MN-REQ-06.14). - -Order is fixed: the caller must confirm memnet-mcp and the gateway already -retry ``serve_draining``. This process then preflights the new interpreter, -drains, swaps ExecStart, updates the gateway pin, starts the new serve, and -rolls back to the unit backup if verification fails. - -No session ids are printed. Tokens are read from the environment. -""" - -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from memnet.exceptions import MemNetError -from memnet.upgrade import ( - MANIFEST_NAME, - RESTORE_NAME, - SUPPORTED_SNAPSHOT_FORMATS, - retire_manifest, -) - -_PROBE = ( - "import json,memnet\n" - "from memnet.upgrade import SUPPORTED_SNAPSHOT_FORMATS\n" - "print(json.dumps({'version': memnet.__version__, " - "'formats': list(SUPPORTED_SNAPSHOT_FORMATS)}))\n" -) - - -class UpgradeRunError(RuntimeError): - def __init__(self, code: str, message: str) -> None: - self.code = code - super().__init__(message) - - -def swap_execstart(text: str, new_exec: str) -> str: - """Replace every ExecStart= line. The previous text is the rollback copy.""" - lines = text.splitlines(keepends=True) - found = False - out: list[str] = [] - for line in lines: - if line.startswith("ExecStart="): - nl = "\n" if line.endswith("\n") else "" - out.append(f"ExecStart={new_exec}{nl}") - found = True - else: - out.append(line) - if not found: - raise UpgradeRunError("upgrade_unit", "unit has no ExecStart") - return "".join(out) - - -def set_pinned_version(config_text: str, product: str, version: str) -> str: - data = json.loads(config_text) - products = data.get("products") - if not isinstance(products, dict) or product not in products: - raise UpgradeRunError("upgrade_pin", "product is not in the gateway config") - row = products[product] - if not isinstance(row, dict): - raise UpgradeRunError("upgrade_pin", "product entry must be an object") - row["pinned_version"] = version - return json.dumps(data, indent=2) + "\n" - - -def preflight_new_python(python: str, directory: Path) -> dict[str, Any]: - """Import the new version and check snapshot format support.""" - proc = subprocess.run( - [python, "-c", _PROBE], - check=False, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - raise UpgradeRunError("upgrade_preflight", "new interpreter failed to import memnet") - try: - info = json.loads(proc.stdout.strip().splitlines()[-1]) - except (json.JSONDecodeError, IndexError) as exc: - raise UpgradeRunError("upgrade_preflight", "new interpreter probe was not JSON") from exc - formats = set(info.get("formats") or []) - manifest_path = directory / MANIFEST_NAME - if manifest_path.is_file(): - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - fmt = int(manifest.get("snapshot_format") or 0) - if fmt not in formats: - raise UpgradeRunError( - "upgrade_snapshot_format", - f"new version cannot load snapshot format {fmt}", - ) - elif snapshot_local_format() not in formats: - raise UpgradeRunError( - "upgrade_snapshot_format", - "new version cannot load the current snapshot format", - ) - return info - - -def snapshot_local_format() -> int: - return int(SUPPORTED_SNAPSHOT_FORMATS[0]) - - -def _read(path: Path) -> str: - return path.read_text(encoding="utf-8") - - -def _write(path: Path, text: str) -> None: - path.write_text(text, encoding="utf-8") - - -def run_upgrade( - *, - new_python: str, - new_exec: str, - state: Path, - clients_ready: bool, - unit_path: Path | None = None, - gateway_config: Path | None = None, - product: str | None = None, - admin_token: str | None = None, - allow_unsaved: bool = False, - restarter: Callable[[], None] | None = None, - drainer: Callable[[], Any] | None = None, - verify_wait_s: float = 30.0, -) -> dict[str, Any]: - """Drain, swap, verify. Roll back the unit and pin if verify fails.""" - if not clients_ready: - raise UpgradeRunError( - "upgrade_clients", - "deploy memnet-mcp and gateway retry before the serve swap", - ) - info = preflight_new_python(new_python, state) - token = admin_token if admin_token is not None else os.environ.get("MEMNET_ADMIN_TOKEN") - if drainer is None: - _drain_via_serve(state, token, allow_unsaved=allow_unsaved) - else: - drainer() - unit_backup: str | None = None - pin_backup: str | None = None - if unit_path is not None: - unit_backup = _read(unit_path) - backup_path = unit_path.with_suffix(unit_path.suffix + ".bak") - _write(backup_path, unit_backup) - _write(unit_path, swap_execstart(unit_backup, new_exec)) - if gateway_config is not None: - if not product: - raise UpgradeRunError("upgrade_pin", "product is required to update the pin") - pin_backup = _read(gateway_config) - pin_path = gateway_config.with_suffix(gateway_config.suffix + ".bak") - _write(pin_path, pin_backup) - _write( - gateway_config, - set_pinned_version(pin_backup, product, str(info["version"])), - ) - if restarter is not None: - restarter() - report = _wait_restore(state, verify_wait_s) - if report is None or int(report.get("failed") or 0) != 0 or report.get("skipped"): - _rollback(unit_path, unit_backup, gateway_config, pin_backup, restarter) - raise UpgradeRunError("upgrade_verify", "restore did not verify; rolled back") - retire_manifest(state) - return {"ok": True, "version": info["version"], "restored": int(report.get("ok") or 0)} - - -def _drain_via_serve(state: Path, token: str | None, *, allow_unsaved: bool) -> None: - from memnet.serve import send_command - - args = ["admin", "upgrade-prepare", "--state-dir", str(state)] - if allow_unsaved: - args.append("--allow-unsaved") - raw = send_command(args, admin_token=token, timeout=120.0) - if int(raw.get("exit_code") or 1) != 0: - raise UpgradeRunError("upgrade_drain", "drain did not reach ready_to_stop") - - -def _wait_restore(state: Path, timeout_s: float) -> dict[str, Any] | None: - deadline = time.monotonic() + timeout_s - path = state / RESTORE_NAME - while time.monotonic() < deadline: - if path.is_file(): - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - time.sleep(0.05) - continue - if isinstance(data, dict): - return data - time.sleep(0.05) - return None - - -def _rollback( - unit_path: Path | None, - unit_backup: str | None, - gateway_config: Path | None, - pin_backup: str | None, - restarter: Callable[[], None] | None, -) -> None: - if unit_path is not None and unit_backup is not None: - _write(unit_path, unit_backup) - if gateway_config is not None and pin_backup is not None: - _write(gateway_config, pin_backup) - if restarter is not None: - restarter() - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="memnet-upgrade") - parser.add_argument("--new-python", required=True) - parser.add_argument("--new-exec", required=True, help="New unit ExecStart value") - parser.add_argument("--state-dir", required=True) - parser.add_argument("--unit", default="") - parser.add_argument("--gateway-config", default="") - parser.add_argument("--product", default="") - parser.add_argument("--allow-unsaved", action="store_true") - parser.add_argument( - "--clients-ready", - action="store_true", - help="Confirm memnet-mcp and the gateway already retry serve_draining", - ) - parser.add_argument("--restart-unit", default="", help="systemctl unit name to restart") - args = parser.parse_args(argv) - restarter = None - if args.restart_unit: - unit_name = args.restart_unit - - def restarter() -> None: - subprocess.run(["systemctl", "restart", unit_name], check=True) - - try: - result = run_upgrade( - new_python=args.new_python, - new_exec=args.new_exec, - state=Path(args.state_dir), - clients_ready=args.clients_ready, - unit_path=Path(args.unit) if args.unit else None, - gateway_config=Path(args.gateway_config) if args.gateway_config else None, - product=args.product or None, - allow_unsaved=args.allow_unsaved, - restarter=restarter, - ) - except (UpgradeRunError, MemNetError) as exc: - code = getattr(exc, "code", "upgrade") - sys.stderr.write(f"@ERR: {code}|{exc}\n") - return 2 - sys.stdout.write(json.dumps({"ok": True, "restored": result["restored"]}) + "\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/parts/memnet-mcp/software/memnet_mcp/client.py b/parts/memnet-mcp/software/memnet_mcp/client.py index dacd183..32e0aa1 100644 --- a/parts/memnet-mcp/software/memnet_mcp/client.py +++ b/parts/memnet-mcp/software/memnet_mcp/client.py @@ -131,18 +131,10 @@ def run_memnet( mode = _transport() if mode == "tcp": - from memnet.upgrade_retry import call_with_upgrade_retry - - def _once() -> dict: - if not probe(): - raise ConnectionRefusedError("serve down") - return send_command(full_argv, stdin=stdin) - - try: - raw = call_with_upgrade_retry(_once) - except (ConnectionError, OSError, TimeoutError): - return MemNetResponse.serve_required(session_hint=session) - return MemNetResponse.from_raw(raw, session_hint=session) + if probe(): + raw = send_command(full_argv, stdin=stdin) + return MemNetResponse.from_raw(raw, session_hint=session) + return MemNetResponse.serve_required(session_hint=session) raw = run_argv(full_argv, stdin=stdin) return MemNetResponse.from_raw(raw, session_hint=session) diff --git a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py index 369e793..6595d68 100644 --- a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py +++ b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py @@ -526,24 +526,29 @@ def _version(self, backend: Backend) -> str | None: ): return hit[1] version: str | None = None - raw = self._command_with_retry(backend, ["version"], None) - if isinstance(raw, dict): - for line in (raw.get("stdout") or "").splitlines(): - if line.startswith("@VER: memnet|"): - version = line.split("|", 1)[1].strip() - break + if probe(host=backend.host, port=backend.port): + try: + raw = send_command( + ["version"], + host=backend.host, + port=backend.port, + timeout=self.timeout_s, + ) + except (OSError, json.JSONDecodeError): + raw = None + if isinstance(raw, dict): + for line in (raw.get("stdout") or "").splitlines(): + if line.startswith("@VER: memnet|"): + version = line.split("|", 1)[1].strip() + break with self._lock: self._ver_cache[backend.id] = (now, version) return version - def _command_with_retry( + def _forward( self, backend: Backend, argv: list[str], stdin: str | None ) -> dict[str, Any] | None: - from memnet.upgrade_retry import call_with_upgrade_retry - - def _once() -> dict[str, Any]: - if not probe(host=backend.host, port=backend.port): - raise ConnectionRefusedError("backend down") + try: raw = send_command( list(argv), stdin=stdin, @@ -551,20 +556,9 @@ def _once() -> dict[str, Any]: port=backend.port, timeout=self.timeout_s, ) - if not isinstance(raw, dict): - raise ConnectionError("bad envelope") - return raw - - try: - return call_with_upgrade_retry(_once) - except (OSError, json.JSONDecodeError, ConnectionError): + except (OSError, json.JSONDecodeError): return None - - def _forward( - self, backend: Backend, argv: list[str], stdin: str | None - ) -> dict[str, Any] | None: - raw = self._command_with_retry(backend, argv, stdin) - if raw is None: + if not isinstance(raw, dict): return None err = (raw.get("stderr") or "").strip() if err == _CLIENT_TIMEOUT and not (raw.get("stdout") or "").strip(): diff --git a/parts/memnet-mcp/software/memnet_mcp/serve_bridge.py b/parts/memnet-mcp/software/memnet_mcp/serve_bridge.py index 5e74fe5..8cd2025 100644 --- a/parts/memnet-mcp/software/memnet_mcp/serve_bridge.py +++ b/parts/memnet-mcp/software/memnet_mcp/serve_bridge.py @@ -12,11 +12,4 @@ def probe(self) -> bool: return probe() def send(self, argv: list[str], *, stdin: str | None = None) -> dict: - from memnet.upgrade_retry import call_with_upgrade_retry - - def _once() -> dict: - if not probe(): - raise ConnectionRefusedError("serve down") - return send_command(argv, stdin=stdin) - - return call_with_upgrade_retry(_once) + return send_command(argv, stdin=stdin) diff --git a/pyproject.toml b/pyproject.toml index c1d4933..3a08449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,6 @@ agensgraph = [ [project.scripts] memnet = "memnet.cli:main" memnet-mcp = "memnet_mcp.server:main" -memnet-upgrade = "memnet.upgrade_run:main" [tool.hatch.version] path = "parts/common/memnet/memnet/__init__.py" diff --git a/tests/conftest.py b/tests/conftest.py index a6bbc3b..6203883 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,12 +8,6 @@ from memnet.session import purge_expired, reset_registry, set_now_override -@pytest.fixture(autouse=True) -def _upgrade_retry_off(monkeypatch: pytest.MonkeyPatch): - """Production default is a 30s upgrade retry. Tests opt in.""" - monkeypatch.setenv("MEMNET_UPGRADE_RETRY_S", "0") - - @pytest.fixture def memnet_temp(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("MEMNET_TEST_INLINE", "1") diff --git a/tests/test_safe_upgrade.py b/tests/test_safe_upgrade.py index f23f103..b4e1222 100644 --- a/tests/test_safe_upgrade.py +++ b/tests/test_safe_upgrade.py @@ -1,4 +1,4 @@ -"""Safe serve upgrade: drain, manifest, restore, retry, rollback (MN-REQ-06.14).""" +"""Safe serve upgrade: drain, manifest, restore, and retire (MN-REQ-06.14).""" from __future__ import annotations @@ -25,9 +25,8 @@ prepare_upgrade, reset_drain_gate, restore_manifest, + startup_restore_or_exit, ) -from memnet.upgrade_retry import call_with_upgrade_retry -from memnet.upgrade_run import UpgradeRunError, run_upgrade, set_pinned_version, swap_execstart TOKEN = "test-admin-token" @@ -151,11 +150,13 @@ def test_corrupt_snapshot_fails_loud_and_keeps_bytes( corrupt = snap.read_bytes() reset_registry() reset_drain_gate() - report = restore_manifest(tmp_path) - assert report.failed == 1 - assert report.ok == 0 + with pytest.raises(SystemExit) as exc: + startup_restore_or_exit(tmp_path) + assert exc.value.code == 3 assert snap.read_bytes() == corrupt assert snap.is_file() + manifest = json.loads((tmp_path / MANIFEST_NAME).read_text(encoding="utf-8")) + assert manifest.get("retired") is not True def test_older_patch_snapshot_loads(memnet_temp, monkeypatch, tmp_path, schema_file): @@ -255,146 +256,28 @@ def test_inflight_blocks_ready_until_finished(): reset_drain_gate() -def test_client_retries_connection_refusal_and_draining(monkeypatch): - monkeypatch.setenv("MEMNET_UPGRADE_RETRY_S", "2") - state = {"n": 0} - - def flaky() -> dict: - state["n"] += 1 - if state["n"] < 3: - raise ConnectionRefusedError("down") - return {"exit_code": 0, "stdout": "", "stderr": ""} - - raw = call_with_upgrade_retry(flaky) - assert raw["exit_code"] == 0 - assert state["n"] == 3 - - state["n"] = 0 - - def draining() -> dict: - state["n"] += 1 - if state["n"] < 3: - return { - "exit_code": 2, - "stdout": "", - "stderr": "@ERR: serve_draining|retry_after_s=5\n", - } - return {"exit_code": 0, "stdout": "ok\n", "stderr": ""} - - raw = call_with_upgrade_retry(draining) - assert raw["stdout"] == "ok\n" - assert state["n"] == 3 - - -def test_client_no_retry_when_window_disabled(monkeypatch): - monkeypatch.setenv("MEMNET_UPGRADE_RETRY_S", "0") - - def down() -> dict: - raise ConnectionRefusedError("down") - - with pytest.raises(ConnectionRefusedError): - call_with_upgrade_retry(down) - - -def test_mcp_tcp_retries_until_serve_accepts(monkeypatch): - monkeypatch.setenv("MEMNET_UPGRADE_RETRY_S", "2") - monkeypatch.setenv("MEMNET_MCP_TRANSPORT", "tcp") - monkeypatch.delenv("MEMNET_TEST_INLINE", raising=False) - calls = {"n": 0} - - def fake_probe(*_a, **_k) -> bool: - calls["n"] += 1 - return calls["n"] >= 3 - - def fake_send(*_a, **_k) -> dict: - return {"exit_code": 0, "stdout": "@STAT: sessions|0|1024\n", "stderr": ""} - - monkeypatch.setattr("memnet_mcp.client.probe", fake_probe) - monkeypatch.setattr("memnet_mcp.client.send_command", fake_send) - from memnet_mcp.client import run_memnet - - resp = run_memnet(["session", "list"]) - assert resp.exit_code == 0 - assert calls["n"] >= 3 - - -def test_rollback_restores_unit_and_pin(memnet_temp, tmp_path): - unit = tmp_path / "memnet-serve.service" - unit.write_text("[Service]\nExecStart=/old/venv/bin/memnet serve\n", encoding="utf-8") - gateway = tmp_path / "gateway.json" - gateway.write_text( - json.dumps({"products": {"endleaf": {"pinned_version": "0.19.19", "backends": []}}}), - encoding="utf-8", - ) - state = tmp_path / "state" - state.mkdir() - calls = {"n": 0} - - def drainer() -> None: - manifest = { - "ready_to_stop": True, - "retired": False, - "snapshot_format": 1, - "sessions": [], - "unsaved": [], - } - (state / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8") - - def restarter() -> None: - calls["n"] += 1 - if calls["n"] == 1: - (state / "upgrade-restore.json").write_text( - json.dumps({"ok": 0, "failed": 1, "skipped": False, "errors": []}), - encoding="utf-8", - ) - - with pytest.raises(UpgradeRunError) as exc: - run_upgrade( - new_python=sys.executable, - new_exec="/new/venv/bin/memnet serve", - state=state, - clients_ready=True, - unit_path=unit, - gateway_config=gateway, - product="endleaf", - drainer=drainer, - restarter=restarter, - verify_wait_s=1, - ) - assert exc.value.code == "upgrade_verify" - assert "ExecStart=/old/venv/bin/memnet serve" in unit.read_text(encoding="utf-8") - pin = json.loads(gateway.read_text(encoding="utf-8")) - assert pin["products"]["endleaf"]["pinned_version"] == "0.19.19" - assert (unit.with_suffix(".service.bak")).is_file() - assert calls["n"] == 2 - - -def test_clients_must_be_ready_before_swap(tmp_path): - unit = tmp_path / "memnet-serve.service" - unit.write_text("ExecStart=/old/venv/bin/memnet serve\n", encoding="utf-8") - with pytest.raises(UpgradeRunError) as exc: - run_upgrade( - new_python=sys.executable, - new_exec="/new/venv/bin/memnet serve", - state=tmp_path, - clients_ready=False, - unit_path=unit, - drainer=lambda: None, - restarter=lambda: None, - ) - assert exc.value.code == "upgrade_clients" - assert unit.read_text(encoding="utf-8").startswith("ExecStart=/old") - - -def test_swap_and_pin_helpers(): - swapped = swap_execstart("ExecStart=/old/bin/memnet serve\n", "/new/bin/memnet serve") - assert swapped == "ExecStart=/new/bin/memnet serve\n" - pinned = set_pinned_version( - json.dumps({"products": {"endleaf": {"pinned_version": "0.19.19"}}}), - "endleaf", - "0.19.20", - ) - assert json.loads(pinned)["products"]["endleaf"]["pinned_version"] == "0.19.20" +def test_clean_restore_retires_so_a_later_start_does_not_replay( + memnet_temp, monkeypatch, tmp_path, schema_file +): + monkeypatch.setenv("MEMNET_ADMIN_TOKEN", TOKEN) + monkeypatch.setenv("MEMNET_STATE_DIR", str(tmp_path)) + ss = open_session(map_file=str(schema_file)) + _mutate(ss.session_id, "CREATE (:TSK {id: 'TSK_once', goal: 'once', status: 'open'})\n") + sid = ss.session_id + assert prepare_upgrade(TOKEN, directory=tmp_path).ready_to_stop is True + reset_registry() + reset_drain_gate() + report = startup_restore_or_exit(tmp_path) + assert report.failed == 0 + assert report.ok == 1 + assert get_entry(sid) is not None + manifest = json.loads((tmp_path / MANIFEST_NAME).read_text(encoding="utf-8")) + assert manifest["retired"] is True + reset_registry() + again = startup_restore_or_exit(tmp_path) + assert again.skipped is True + assert get_entry(sid) is None + assert (tmp_path / "upgrade-snapshots").is_dir() def _free_port() -> int: @@ -426,7 +309,6 @@ def test_tcp_restart_restores_same_sessions(memnet_temp, monkeypatch, tmp_path, "MEMNET_SERVE_HOST": "127.0.0.1", "MEMNET_SERVE_PORT": str(port), "MEMNET_MAX_ROWS": "80", - "MEMNET_UPGRADE_RETRY_S": "0", } ) env.pop("MEMNET_TEST_INLINE", None) @@ -545,6 +427,20 @@ def stop(proc: subprocess.Popen[bytes]) -> None: assert "kept-fact" in found["stdout"] assert (state / "upgrade-snapshots").is_dir() assert list((state / "upgrade-snapshots").glob("*.snap")) + manifest = json.loads((state / MANIFEST_NAME).read_text(encoding="utf-8")) + assert manifest["retired"] is True + finally: + stop(proc) + + proc = start() + try: + replayed = send_command( + ["query", "find", "--kind", "TSK", "--limit", "5", "--session", sid], + host="127.0.0.1", + port=port, + ) + assert replayed["exit_code"] != 0 + assert "kept-fact" not in replayed["stdout"] finally: stop(proc) @@ -554,3 +450,7 @@ def test_upgrade_doc_has_no_session_id(): encoding="utf-8" ) assert "mn_" not in text + assert "memnet-upgrade" not in text + assert "MEMNET_UPGRADE_RETRY_S" not in text + assert "upgrade-prepare" in text + assert "ready_to_stop" in text or "ready-to-stop" in text