From 8e60e0d29cd498404811d0e4a2d9de5e8750cf2d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 15:27:41 +0000 Subject: [PATCH 1/3] Add the memnet-mcp product gateway (MN-REQ-06.12). Route by product, house, or session to one owning serve, with hashed per-product credentials and verbatim serve pass-through. Single-backend stdio and streamable-http stay unchanged when no registry is set. No SemVer bump. Co-authored-by: chouswei --- .env.example | 7 + CHANGELOG.md | 1 + docs/README.md | 1 + docs/operations/README.md | 1 + docs/operations/product-gateway-contract.md | 279 ++++++ parts/memnet-mcp/README.md | 3 + .../software/memnet_mcp/product_gateway.py | 922 ++++++++++++++++++ .../memnet-mcp/software/memnet_mcp/server.py | 13 +- sysml-models/README.md | 7 +- sysml-models/config.yaml | 4 +- sysml-models/models/deploy.sysml | 62 +- sysml-models/models/implementation.sysml | 22 +- sysml-models/models/requirements.sysml | 61 ++ sysml-models/models/verify.sysml | 54 + sysml-models/outputs/README.md | 1 + .../outputs/product-gateway-case-study.md | 27 + sysml-models/outputs/product-nest-one-page.md | 4 +- .../outputs/ssot-to-code-allocate-map.md | 1 + tests/test_product_gateway.py | 495 ++++++++++ tests/test_sysml_product_gateway.py | 68 ++ 20 files changed, 2020 insertions(+), 13 deletions(-) create mode 100644 docs/operations/product-gateway-contract.md create mode 100644 parts/memnet-mcp/software/memnet_mcp/product_gateway.py create mode 100644 sysml-models/outputs/product-gateway-case-study.md create mode 100644 tests/test_product_gateway.py create mode 100644 tests/test_sysml_product_gateway.py diff --git a/.env.example b/.env.example index 9fa511ed..6519c080 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,13 @@ MEMNET_SESSION_TTL_MINUTES=60 # TCP frame size cap (default 4 MiB): # MEMNET_SERVE_MAX_FRAME_BYTES=4194304 +# Product gateway (opt-in). Unset = single-backend memnet-mcp, unchanged. +# JSON maps backend id -> host:port, product backends, pinned version, and +# sha256 of each product credential. No secrets in this file. +# MEMNET_GATEWAY_CONFIG=/var/lib/memnet/gateway.json +# Public bind is refused unless this is 1. Leave unset for Endleaf. +# MEMNET_GATEWAY_ALLOW_PUBLIC= + # Optional CheapLlmImportGuard (Path-B import-slice soft LLM). Unset = off; absorb still works. # Set the key on the serve process to activate. Not MEMNET_MCP_HTTP_TOKEN / session_token / RSV llm_id. # MEMNET_IMPORT_GUARD_API_KEY= diff --git a/CHANGELOG.md b/CHANGELOG.md index ab6e6331..de9592b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ## [Unreleased] ### Added +- **Product gateway on memnet-mcp (MN-REQ-06.12)** — `memnet-mcp --transport gateway` routes by product, house, or session to one owning `memnet serve`. Hashed per-product credentials, namespace isolation, verbatim GQL / stdin / `@ERR` `@WRN` `@STAT` pass-through, gateway refusals, health with failover only for a new unpinned session, version pin, loopback or tailnet bind, per-product admin counts. Single-backend stdio and streamable-http stay unchanged when no registry is configured. No SemVer bump. Not deployed. Wire: [`docs/operations/product-gateway-contract.md`](docs/operations/product-gateway-contract.md). - **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe and tests; no engine or cap-default change. No SemVer bump. Wire: [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md). ### Changed diff --git a/docs/README.md b/docs/README.md index fb2b62fd..90363baf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ Multitask MUST for this product. Index: [`operations/README.md`](operations/READ | [`operations/honesty-c-wire-audit.md`](operations/honesty-c-wire-audit.md) | 0.19.10 CueMiss/Peak_L vs CueConflict; snapshot locator SCHEMA warn; Path-B SysML CON; Truncation; hid emit audit | | [`operations/tip-memnet-access-portal.md`](operations/tip-memnet-access-portal.md) | Sidecar: keyed tip MemNet MCP access (invite, Google, Bearer; tip≠face; not sysmledge) | | [`operations/memnet-lan-mcp-front.md`](operations/memnet-lan-mcp-front.md) | Later invent #191: ClusterRoute — one MCP catalogue over N LAN serves (tip≠face; not shipped) | +| [`operations/product-gateway-contract.md`](operations/product-gateway-contract.md) | Product gateway on memnet-mcp (MN-REQ-06.12): route by product, house, or session | | [`operations/cluster-route-vs-slice-hand-carry.md`](operations/cluster-route-vs-slice-hand-carry.md) | Two named moves: ClusterRoute vs SliceHandCarry (#191 / #47 cousin; inventOnly) | | [`operations/admin-usage-report.md`](operations/admin-usage-report.md) | Admin-only serve usage JSON for a product-gate admin MCP (opaque alias; not agent MCP) | | [`operations/one-session-per-document.md`](operations/one-session-per-document.md) | One MemNet session per document over loopback serve (no MCP front) | diff --git a/docs/operations/README.md b/docs/operations/README.md index 043d7132..4bf2e9c0 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -9,6 +9,7 @@ Agent operating doctrine for this product (not domain recipes). | [`../cap-contract.md`](../cap-contract.md) | Product-gate cap contract: every refuse/clip, exact `@ERR` / Truncation, integrator checklist | | [`tip-memnet-access-portal.md`](tip-memnet-access-portal.md) | Live sidecar: admin invite, Google login, Bearer for keyed tip MCP (tip≠face; not sysmledge) | | [`memnet-lan-mcp-front.md`](memnet-lan-mcp-front.md) | Later invent #191: ClusterRoute — one MCP catalogue, N LAN serves (tip≠face; not shipped) | +| [`product-gateway-contract.md`](product-gateway-contract.md) | Product gateway on memnet-mcp (MN-REQ-06.12): per-product route to one owning serve | | [`cluster-route-vs-slice-hand-carry.md`](cluster-route-vs-slice-hand-carry.md) | Two named moves: ClusterRoute vs SliceHandCarry (#191 / #47 cousin; inventOnly) | | [`admin-usage-report.md`](admin-usage-report.md) | Admin-only serve usage JSON (opaque alias; not agent MCP; MN-REQ-06.11) | | [`one-session-per-document.md`](one-session-per-document.md) | Product gate: one serve session per document over loopback (no MCP); 0.19.18 probe | diff --git a/docs/operations/product-gateway-contract.md b/docs/operations/product-gateway-contract.md new file mode 100644 index 00000000..c8f9f985 --- /dev/null +++ b/docs/operations/product-gateway-contract.md @@ -0,0 +1,279 @@ +# Product gateway contract + +**Status:** implemented for the behaviours in [This cut](#this-cut). **MN-REQ-06.12** / **MN-VER-06-S10**. Model: `MemNetProductGateway` nested on `MemNetLanMcpFront` (`sysml-models/models/deploy.sysml`). Code: `parts/memnet-mcp/software/memnet_mcp/product_gateway.py`. No SemVer bump. + +The gateway is `memnet-mcp --transport gateway`. It is not a separate droplet shim. The #191 catalogue (MCP tool union, SnapshotHandCarry) stays invent-only on the parent part. + +Online products reach a `memnet serve` only through this process. Endleaf now. Atelier and SysMLEdge later, each with their own credential and backend list. + +## Today + +Checked on tags **v0.19.6** (the droplet `memnet-mcp-http`) and **v0.19.18** (the Pi serve). This checkout matches tag v0.19.18 for the MCP and serve files cited below. + +### (1) One MCP, one serve + +Neither tag can point one MCP process at several serves. + +| Tag | How the backend is chosen | +|-----|---------------------------| +| v0.19.6 | `MEMNET_MCP_TRANSPORT` is `inprocess` (default) or `tcp` / `serve`. TCP calls `probe()` and `send_command()` with no host argument, so both use `MEMNET_SERVE_HOST` (default `127.0.0.1`) and `MEMNET_SERVE_PORT` (default `18765`). | +| v0.19.18 | Same. The only client change is that `session expire-status` is left off the session flag. | + +Citations: + +- v0.19.6 `parts/memnet-mcp/software/memnet_mcp/client.py` lines 105–139 (`_transport`, `run_memnet`) +- v0.19.6 `parts/common/memnet/memnet/config.py` lines 90–95 (`serve_host`, `serve_port`) +- v0.19.18 `parts/memnet-mcp/software/memnet_mcp/client.py` lines 106–140 +- v0.19.18 `parts/common/memnet/memnet/config.py` lines 96–101 + +There is no backend id, no product table, and no second host. Setting `MEMNET_SERVE_HOST=100.118.79.40` and `MEMNET_SERVE_PORT=18795` would aim the single TCP client at the Endleaf Pi. It would not route a second product elsewhere. + +### (2) What auth exists in front + +| Check | Where | What is checked | +|-------|--------|-----------------| +| Optional shared bearer | streamable-http only | `MEMNET_MCP_HTTP_TOKEN`. Exact `Authorization: Bearer `. Empty or unset means auth off. | +| stdio | default `memnet-mcp` | No bearer, no tip key, no product credential. | +| `caller` | MCP tool argument, both tags | Optional `--caller` on pin_map / mutate / find when that session's CapsPolicy ACL is enabled. Not a gateway. Off unless the session turned ACL on. | +| Tip `mn_tip_…` | v0.19.18 portal only. Absent at v0.19.6. | SHA-256 of the key, invite not revoked. nginx `auth_request` in front of one public `/mcp`. Not scoped to a product's backends. Not a version pin. Not inside `memnet-mcp`. | + +Citations: + +- v0.19.6 and v0.19.18 `parts/memnet-mcp/software/memnet_mcp/http_transport.py` lines 70–76 and 230–264 (`mcp_http_token`, `SharedBearerASGI`). The file is identical across the two tags. +- v0.19.6 `parts/memnet-mcp/software/memnet_mcp/server.py` lines 233–234 (`--caller` only when the tool argument is set). The same argument exists on v0.19.18. +- v0.19.6 has no `ops/tip_access_portal/`. +- v0.19.18 `ops/tip_access_portal/tip_access_portal/store.py` lines 22–23 (`hash_secret`), 158–161 (`mn_tip_` + SHA-256 at insert), 206–210 (`key_is_active`) +- v0.19.18 `ops/tip_access_portal/tip_access_portal/gate.py` lines 8–12 + +**Reuse decision.** The portal already hashes and revokes a bearer. It does not map a key to a backend id, a house, or a pinned serve version. This gateway copies that hash-and-revoke pattern into its own config. It does not call the portal store and it does not accept an `mn_tip_` key as a product credential. `MEMNET_MCP_HTTP_TOKEN` stays the streamable-http shared secret and is not a product credential either. + +### (3) Can a 0.19.6 MCP talk to a 0.19.18 serve? + +Yes, for the argv that v0.19.6 MCP actually sends. The length-prefixed JSON envelope is the same shape. + +| | v0.19.6 | v0.19.18 | +|--|---------|----------| +| Request | `{"args": [...]}` plus optional `"stdin"` (`serve.py` `send_command` lines 206–217; `_handle_request` lines 76–108) | Same, plus optional `admin_token` and `admin_usage` only when the client sets them (`serve.py` lines 81–126 and 263–282). v0.19.6 never sets them. | +| Frame | 4-byte big-endian length + JSON | Same | +| Reply | `{exit_code, stdout, stderr}` | Same. `parse.py` is identical across the tags, so session-id and `@ERR` extraction do not change. | +| Version line | `@VER: memnet\|` (`cli.py` line 215) | Same shape (`cli.py` line 225). The text is `0.19.6` or `0.19.18`. | + +v0.19.18 CLI adds optional flags `--max-edges`, `--product`, and `--token`. It removes none. New commands the old MCP does not send: `admin usage-report`, `session expire-status`. + +A v0.19.6 MCP `session open`, `pin_map`, `mutate`, `session load --file`, and `session save` argv is still accepted. The old MCP cannot select a backend other than the one `MEMNET_SERVE_HOST` / `MEMNET_SERVE_PORT`. + +The other direction is not the same. A v0.19.18 MCP `snap_model` / `ingest_*` call always sends `--max-edges`. A v0.19.6 serve CLI has no such flag and will refuse that argv. That does not block a 0.19.6 MCP talking to a 0.19.18 serve. + +## This cut + +`memnet-mcp --transport gateway` reads `MEMNET_GATEWAY_CONFIG` (a JSON file). stdio and streamable-http do not read it. With no registry, one memnet-mcp process behaves as it does today. + +### Endpoint + +`POST ` (default `/gateway`) with `Content-Type: application/json` and: + +```http +Authorization: Bearer +``` + +The JSON is the serve envelope plus routing fields. Only `args` and `stdin` are forwarded. `session`, `namespace`, `product`, `house`, and `backend` are not sent to the serve. `admin_token` in the body is dropped, so a product credential cannot call the backend admin report. + +```json +{ + "args": ["mutate", "--stdin", "--session", "mn_…"], + "stdin": "MATCH (n {id: 'TSK_gw'}) SET n.status = 'settled'\n", + "session": "mn_…", + "namespace": "endleaf", + "house": "syson" +} +``` + +`namespace` or `product`, when present, must equal the product of the credential. `session`, when present both in the field and in `args`, must be the same string. The gateway does not insert `--session` into `args`. + +`house` is a name in that product's `houses` map (a backend id). `backend` is an explicit backend id in that product's list. Either one pins placement of a **new** session. If the pinned backend is down or the version does not match, the gateway refuses. It does not fail over away from a pin. + +With neither `house` nor `backend`, a new session walks `products..backends` in order and uses the first that accepts TCP and whose `memnet version` line equals `pinned_version`. + +A later call for a known session goes to the owning backend only. A `house` or `backend` that names a different backend is `gateway_forbidden_session`. The session is not moved. + +### Reply + +```json +{"exit_code": 0, "stdout": "…", "stderr": "…"} +``` + +Engine `exit_code`, stdout, and stderr are the backend's bytes, including `@ERR`, `@WRN`, and `@STAT`, with one exception: `session list` drops `@SESSION` rows this product does not own and rewrites `@STAT: sessions|n/max` so `n` is the filtered count. `max` stays the backend's figure (the first successful backend's max). Other control lines are copied. If any allowed backend is down or the pin does not match, stderr also carries a `gateway_` line and `exit_code` is 2, so the list does not look complete. + +HTTP status is a hint. The body is always the envelope. + +| HTTP | When | +|------|------| +| 200 | Backend reply, including engine `exit_code` other than 0 | +| 401 | `gateway_auth` | +| 403 | `gateway_forbidden_session` or `gateway_forbidden` | +| 409 | `gateway_backend_version_mismatch` | +| 413 | `gateway_body_too_large` | +| 502 | `gateway_backend_unreachable` | + +### Gateway errors + +```text +@ERR: gateway_| +``` + +| Code | Meaning | +|------|---------| +| `gateway_auth` | Missing, unknown, or revoked bearer, or namespace does not match the credential | +| `gateway_forbidden_session` | Unknown session, another product's session, or a pin that would move the owner. The detail does not say which. | +| `gateway_forbidden` | `house` or `backend` is outside the product, or argv is `admin` or `serve` | +| `gateway_backend_unreachable` | TCP probe failed, the version command did not answer, or the client wait expired | +| `gateway_backend_version_mismatch` | `memnet version` is not the product pin | +| `gateway_body_too_large` | Body above `body_max_bytes` | +| `gateway_bad_request` | JSON, `args`, or disagreeing session fields | +| `gateway_unconfigured` | Admin counts requested and no admin hash is set; also process exit when `MEMNET_GATEWAY_CONFIG` is missing | + +`exit_code` on these refusals is 2. They are not engine lines. An engine `@ERR: not_found|…` stays `not_found`. + +### Limits + +The gateway does not enforce `MEMNET_MAX_ROWS`, `MEMNET_MAX_BATCH_LINES`, or `MEMNET_MAX_VALUE_BYTES`. The Endleaf Pi runs 0.19.18 with `MEMNET_MAX_ROWS=10000`. Engine defaults elsewhere are `MEMNET_MAX_ROWS` 5000, batch 1000 lines, value 4096 decoded bytes (`parts/common/memnet/memnet/config.py`). A product may send `--max-rows 10000` and a mutate stdin up to the body ceiling. The backend still applies its own caps. Those refusals come back verbatim. + +The gateway body ceiling is `body_max_bytes`, default **4194304** (4 MiB), the same figure as `MEMNET_SERVE_MAX_FRAME_BYTES`. State it in the config. Leave the gateway process's `MEMNET_SERVE_MAX_FRAME_BYTES` at least that large so the TCP client does not refuse a body the contract already accepted. + +### Version pin + +Each product has `pinned_version` (for Endleaf, `0.19.18`). Before a forward, the gateway calls `version` on that backend and requires `@VER: memnet|`. A mismatch is `gateway_backend_version_mismatch` and the argv is not sent. Upgrading the Pi serve without changing the pin refuses the product. Changing the pin is how the owner is notified: edit the config and restart the gateway. + +`version_cache_s` defaults to 15. Set `0` to check every call. A serve upgraded inside a non-zero window can still be reached until the cache expires. + +### Admin counts + +`GET /admin/counts` with `Authorization: Bearer `. + +The admin secret is stored as `admin.sha256` only. The JSON lists each product's `requests`, `gateway_refusals`, and `backend_errors`. It has no graph text and no session id. Counts are in memory and reset when the process stops. + +### Listener + +Default bind `127.0.0.1`. Tailnet addresses in `100.64.0.0/10` (for example `100.118.79.40`) and Tailscale IPv6 `fd7a:115c:a1e0::/48` are allowed. `0.0.0.0` and other public addresses are refused unless `MEMNET_GATEWAY_ALLOW_PUBLIC=1`. Endleaf on the droplet should not set that. Bind the droplet loopback, or the droplet's own tailnet address. Do not publish the port. + +## Examples + +Credential and session ids below are placeholders. + +### session open + +```http +POST /gateway +Authorization: Bearer +Content-Type: application/json + +{ + "args": ["session", "open", "--map-file", "/var/memnet/schema.sysml.example.txt"], + "namespace": "endleaf", + "house": "syson" +} +``` + +```json +{ + "exit_code": 0, + "stdout": "@SESSION: mn_example|2026-10-08T16:00:00Z|60\n", + "stderr": "MEMNET_SESSION=mn_example\n" +} +``` + +The gateway records `mn_example` → backend `endleaf-rpi5-syson`. The map path is on the Pi, because `args` are not rewritten. + +### MATCH … SET + +```json +{ + "args": ["mutate", "--stdin", "--session", "mn_example"], + "stdin": "MATCH (n {id: 'TSK_gw'}) SET n.status = 'settled'\n", + "session": "mn_example", + "namespace": "endleaf" +} +``` + +Reply `exit_code`, stdout, and stderr are the Pi's. A missing element comes back as the engine line `@ERR: not_found|…`, not a `gateway_` line. + +### Edge delete + +On 0.19.18 the form that reaches an edge delete is: + +```json +{ + "args": ["mutate", "--stdin", "--session", "mn_example"], + "stdin": "MATCH (n WHERE true)-[r {id: 'E_gw'}]->() DELETE r\n", + "session": "mn_example" +} +``` + +`MATCH ()-[r {id: 'E_gw'}]-() DELETE r` is still forwarded byte for byte. On this engine it lowers as a node delete and the Pi replies `@ERR: not_found|DELETE matched no element`. The gateway does not rewrite it. See [Open decisions](#open-decisions). + +## Endleaf configuration + +Pi serve (already running, not changed by this repo): `rpi5-syson`, tailnet `100.118.79.40:18795`, MemNet 0.19.18, `MEMNET_MAX_ROWS=10000`, no token and no ACL, accepting loopback and the droplet. + +On the droplet, hash the product credential and an admin credential. Do not commit the plaintext. + +```bash +python -c "import hashlib,sys; print(hashlib.sha256(sys.argv[1].encode()).hexdigest())" 'REPLACE_WITH_ENDLEAF_CREDENTIAL' +``` + +`/etc/memnet/gateway.json` (mode 0600, not in git): + +```json +{ + "bind": "127.0.0.1", + "port": 18770, + "path": "/gateway", + "body_max_bytes": 4194304, + "state_path": "/var/lib/memnet/gateway-owners.json", + "version_cache_s": 0, + "backend_timeout_s": 30, + "admin": {"sha256": ""}, + "backends": { + "endleaf-rpi5-syson": { + "host": "100.118.79.40", + "port": 18795 + } + }, + "products": { + "endleaf": { + "backends": ["endleaf-rpi5-syson"], + "houses": {"syson": "endleaf-rpi5-syson"}, + "pinned_version": "0.19.18", + "credentials": [ + {"id": "endleaf-1", "sha256": "", "revoked": false} + ] + } + } +} +``` + +```bash +export MEMNET_GATEWAY_CONFIG=/etc/memnet/gateway.json +memnet-mcp --transport gateway +``` + +Rotate by appending a new `{id, sha256, revoked: false}` and setting the old object's `revoked` to true. Restart is required for a config edit. The owner file remembers session → backend across restarts. It holds session ids. The admin counts endpoint does not. + +To add a standby later, append another backend id to `backends` and to `endleaf.backends`, in failover order. Do not put Atelier's backend on Endleaf's list. + +The droplet's existing 0.19.6 `memnet-mcp-http` does not speak this contract. This cut is not deployed. + +## Open decisions + +1. **Agent MCP tools.** stdio and streamable-http `pin_map` / `mutate` do not use the registry. Only `--transport gateway` does. Whether those tools should route by session is open. +2. **One gateway process.** The owner file is local. A second gateway does not share it. Multi-instance ownership is open. +3. **Admin counts reset** when the process stops. Durable counts are open. +4. **List `max`.** The filtered stat keeps the backend's `max`. Whether to hide `max` is open. +5. **Unknown and foreign sessions** share `gateway_forbidden_session` so the caller cannot tell them apart. +6. **Failover** is only for a new session with no `house` and no `backend`. There is no automatic return to a recovered preferred backend, and no SnapshotHandCarry (same sid, new owner). That relocate stays the #191 invent. +7. **Public bind** exists only behind `MEMNET_GATEWAY_ALLOW_PUBLIC=1`. Endleaf should leave it unset. +8. **Bind host** must be an IP in loopback or tailnet, or the name `localhost`. MagicDNS names are not resolved. +9. **Tip keys.** Federating `mn_tip_` into this credential store is open. This cut does not. +10. **Version cache.** Default 15 seconds. Endleaf's sample config sets `0`. +11. **Empty-paren edge delete** (`MATCH ()-[r {id}]->() DELETE r`) does not delete an edge on 0.19.18. Fixing that engine lower is open. The gateway will not rewrite it. +12. **`session list` on a shared backend** shows only sids this gateway recorded for that product. A session opened on the Pi by a path other than this gateway is invisible here and is refused as unknown. Whether the gateway should adopt a sid the product already holds on its pinned backend is open. diff --git a/parts/memnet-mcp/README.md b/parts/memnet-mcp/README.md index 0d6b2e96..cd90efc3 100644 --- a/parts/memnet-mcp/README.md +++ b/parts/memnet-mcp/README.md @@ -29,6 +29,9 @@ empty q = outline; ingest is not export). See `docs/grammar/gql-wire-profile.md` |-----------|------|---------| | **stdio** (default) | Local Cursor `command` | `memnet-mcp` | | **streamable-http** (opt-in) | Remote Cursor `"url"` | `memnet-mcp --transport streamable-http` | +| **gateway** (opt-in) | Product HTTP in front of N serves | `memnet-mcp --transport gateway` | + +`--transport gateway` is the product gateway (MN-REQ-06.12). It is off unless `MEMNET_GATEWAY_CONFIG` points at a registry JSON. stdio and streamable-http ignore that file and keep today's single backend (`MEMNET_SERVE_HOST` / `MEMNET_SERVE_PORT`). Contract: [`docs/operations/product-gateway-contract.md`](../../docs/operations/product-gateway-contract.md). HTTP is **not** the default. Doctrine remains **in-process first**; HTTP is for a dedicated remote MCP endpoint (e.g. Pi) without touching Inventree on `:80` / `:443`. diff --git a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py new file mode 100644 index 00000000..6595d68c --- /dev/null +++ b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py @@ -0,0 +1,922 @@ +"""MemNet product gateway — a memnet-mcp capability (MN-REQ-06.12). + +One process routes a serve envelope to the owning ``memnet serve``. +Products never open a second product's sessions. Args and stdin are +forwarded unchanged. Engine ``@ERR`` / ``@WRN`` / ``@STAT`` lines and +``exit_code`` come back as the backend wrote them, except the session +list membership filter. + +When ``MEMNET_GATEWAY_CONFIG`` is unset, this module is not started. +stdio and streamable-http memnet-mcp stay single-backend. +""" + +from __future__ import annotations + +import hashlib +import hmac +import ipaddress +import json +import os +import re +import sys +import threading +import time +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from memnet.serve import is_loopback_bind_host, probe, send_command + +DEFAULT_BIND = "127.0.0.1" +DEFAULT_PORT = 18770 +DEFAULT_PATH = "/gateway" +DEFAULT_BODY_MAX = 4 * 1024 * 1024 +DEFAULT_TIMEOUT_S = 30.0 +DEFAULT_VERSION_CACHE_S = 15.0 +_CGNAT = ipaddress.ip_network("100.64.0.0/10") +_TAILSCALE_V6 = ipaddress.ip_network("fd7a:115c:a1e0::/48") +_STAT_SESSIONS = re.compile(r"^@STAT:\s*sessions\|(\d+)/(\d+)\s*$") +_CLIENT_TIMEOUT = "@ERR: serve_timeout|wait exceeded" +_PLACE_CMDS = frozenset({"open"}) +_REFUSED_ARGV0 = frozenset({"admin", "serve"}) + + +class GatewayConfigError(ValueError): + """Registry file is not usable. The listener does not start.""" + + +class GatewayBindError(RuntimeError): + """Bind host is public, or is not loopback or tailnet.""" + + +@dataclass +class HandleResult: + http_status: int + exit_code: int + stdout: str + stderr: str + + def envelope(self) -> dict[str, Any]: + return { + "exit_code": self.exit_code, + "stdout": self.stdout, + "stderr": self.stderr, + } + + +@dataclass +class Backend: + id: str + host: str + port: int + + +@dataclass +class Credential: + id: str + sha256: str + revoked: bool = False + + +@dataclass +class Product: + id: str + backends: list[str] + houses: dict[str, str] + pinned_version: str + credentials: list[Credential] + + +@dataclass +class OwnerRow: + product: str + backend: str + + +@dataclass +class Counts: + requests: int = 0 + gateway_refusals: int = 0 + backend_errors: int = 0 + + +@dataclass +class ProductGateway: + """In-process router. ``start`` listens; ``handle`` is the contract.""" + + backends: dict[str, Backend] + products: dict[str, Product] + bind_host: str + bind_port: int + path: str + body_max_bytes: int + state_path: str + admin_sha256: str | None + timeout_s: float = DEFAULT_TIMEOUT_S + version_cache_s: float = DEFAULT_VERSION_CACHE_S + owners: dict[str, OwnerRow] = field(default_factory=dict) + counts: dict[str, Counts] = field(default_factory=dict) + _lock: threading.Lock = field(default_factory=threading.Lock) + _ver_cache: dict[str, tuple[float, str | None]] = field(default_factory=dict) + _httpd: ThreadingHTTPServer | None = field(default=None, repr=False) + _thread: threading.Thread | None = field(default=None, repr=False) + + @classmethod + def load( + cls, + path: str, + *, + bind_host: str | None = None, + bind_port: int | None = None, + ) -> ProductGateway: + try: + raw = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise GatewayConfigError(f"gateway config: {exc}") from exc + if not isinstance(raw, dict): + raise GatewayConfigError("gateway config must be a JSON object") + return cls.from_dict(raw, bind_host=bind_host, bind_port=bind_port) + + @classmethod + def from_dict( + cls, + raw: dict[str, Any], + *, + bind_host: str | None = None, + bind_port: int | None = None, + ) -> ProductGateway: + backends = _parse_backends(raw.get("backends")) + products = _parse_products(raw.get("products"), backends) + host = bind_host if bind_host is not None else str(raw.get("bind") or DEFAULT_BIND) + port = bind_port if bind_port is not None else int(raw.get("port") or DEFAULT_PORT) + path = str(raw.get("path") or DEFAULT_PATH) + if not path.startswith("/"): + path = "/" + path + body_max = int(raw.get("body_max_bytes") or DEFAULT_BODY_MAX) + if body_max < 1: + raise GatewayConfigError("body_max_bytes must be positive") + state_path = str(raw.get("state_path") or "").strip() + if not state_path: + raise GatewayConfigError("state_path is required") + admin = raw.get("admin") or {} + admin_sha = None + if isinstance(admin, dict) and admin.get("sha256"): + admin_sha = _hex64(str(admin["sha256"]), what="admin.sha256") + timeout_s = float(raw.get("backend_timeout_s") or DEFAULT_TIMEOUT_S) + version_cache_s = float( + raw["version_cache_s"] if "version_cache_s" in raw else DEFAULT_VERSION_CACHE_S + ) + gw = cls( + backends=backends, + products=products, + bind_host=host, + bind_port=port, + path=path.rstrip("/") or "/", + body_max_bytes=body_max, + state_path=state_path, + admin_sha256=admin_sha, + timeout_s=timeout_s, + version_cache_s=version_cache_s, + counts={pid: Counts() for pid in products}, + ) + gw._load_owners() + return gw + + @property + def admin_path(self) -> str: + return self.path.rstrip("/") + "/admin/counts" + + @property + def bound_port(self) -> int: + if self._httpd is not None: + return int(self._httpd.server_address[1]) + return self.bind_port + + def start(self) -> None: + validate_gateway_bind(self.bind_host) + handler = _handler_factory(self) + self._httpd = ThreadingHTTPServer((self.bind_host, self.bind_port), handler) + self._thread = threading.Thread( + target=self._httpd.serve_forever, + name="memnet-product-gateway", + daemon=True, + ) + self._thread.start() + + def serve_forever(self) -> None: + validate_gateway_bind(self.bind_host) + handler = _handler_factory(self) + httpd = ThreadingHTTPServer((self.bind_host, self.bind_port), handler) + self._httpd = httpd + host, port = httpd.server_address[:2] + names = ",".join(sorted(self.products)) + sys.stderr.write(f"MEMNET_GATEWAY={host}:{port}{self.path} products={names}\n") + sys.stderr.flush() + try: + httpd.serve_forever() + finally: + httpd.server_close() + + def stop(self) -> None: + if self._httpd is not None: + self._httpd.shutdown() + self._httpd.server_close() + self._httpd = None + if self._thread is not None: + self._thread.join(timeout=2) + self._thread = None + + def counts_public(self) -> dict[str, Any]: + with self._lock: + products = { + pid: { + "requests": self.counts[pid].requests, + "gateway_refusals": self.counts[pid].gateway_refusals, + "backend_errors": self.counts[pid].backend_errors, + } + for pid in self.products + } + return {"products": products} + + def handle( + self, + *, + method: str, + path: str, + authorization: str | None, + body: bytes, + ) -> HandleResult: + route = urlsplit(path).path + if method == "GET" and route == self.admin_path: + return self._admin(authorization) + if method != "POST" or route != self.path: + return _gateway("bad_request", "POST " + self.path, http=404) + if len(body) > self.body_max_bytes: + return _gateway( + "body_too_large", + f"{len(body)} bytes exceeds {self.body_max_bytes}", + http=413, + ) + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return _gateway("bad_request", "body must be a JSON object", http=400) + if not isinstance(payload, dict): + return _gateway("bad_request", "body must be a JSON object", http=400) + token = _bearer(authorization) + product = self._product_for_token(token) if token else None + if product is None: + return _gateway("auth", "bearer required", http=401) + self._bump(product.id, requests=1) + result = self._dispatch(product, payload) + if result.stderr.startswith("@ERR: gateway_"): + self._bump(product.id, refusals=1) + elif result.exit_code != 0: + self._bump(product.id, backend_errors=1) + return result + + def _admin(self, authorization: str | None) -> HandleResult: + if not self.admin_sha256: + return _gateway("unconfigured", "admin counts", http=404) + token = _bearer(authorization) + digest = hashlib.sha256(token.encode()).hexdigest() if token else "" + if not token or not hmac.compare_digest(digest, self.admin_sha256): + return _gateway("auth", "admin bearer required", http=401) + body = json.dumps(self.counts_public(), ensure_ascii=False) + return HandleResult(200, 0, body, "") + + def _dispatch(self, product: Product, payload: dict[str, Any]) -> HandleResult: + namespace = payload.get("namespace", payload.get("product")) + if namespace is not None and namespace != product.id: + return _gateway("auth", "credential is not scoped to this namespace", http=401) + args = payload.get("args") + if not isinstance(args, list) or not all(isinstance(item, str) for item in args): + return _gateway("bad_request", "args must be a list of strings", http=400) + argv = list(args) + stdin = payload.get("stdin", None) + if stdin is not None and not isinstance(stdin, str): + return _gateway("bad_request", "stdin must be a string", http=400) + if not argv or argv[0] in _REFUSED_ARGV0: + return _gateway("forbidden", "command is not forwarded", http=403) + try: + session = _session_of(payload, argv) + except ValueError as exc: + return _gateway("bad_request", str(exc), http=400) + kind = _kind(argv) + house = payload.get("house") + backend_pin = payload.get("backend") + if house is not None and not isinstance(house, str): + return _gateway("bad_request", "house must be a string", http=400) + if backend_pin is not None and not isinstance(backend_pin, str): + return _gateway("bad_request", "backend must be a string", http=400) + if kind == "list": + return self._list(product, argv, stdin) + if kind == "place": + return self._place(product, argv, stdin, house=house, backend_pin=backend_pin) + if kind == "probe": + return self._probe(product, argv, stdin, house=house, backend_pin=backend_pin) + if not session: + return _gateway("bad_request", "session required", http=400) + return self._owned( + product, + argv, + stdin, + session, + house=house, + backend_pin=backend_pin, + ) + + def _place( + self, + product: Product, + argv: list[str], + stdin: str | None, + *, + house: str | None, + backend_pin: str | None, + ) -> HandleResult: + chosen, err = self._select_new(product, house=house, backend_pin=backend_pin) + if err is not None or chosen is None: + return err or _gateway("backend_unreachable", "no backend", http=502) + raw = self._forward(chosen, argv, stdin) + if raw is None: + return _gateway("backend_unreachable", chosen.id, http=502) + result = _from_backend(raw) + if result.exit_code == 0: + for sid in _sids(result.stdout, result.stderr): + self._remember(sid, product.id, chosen.id) + return result + + def _probe( + self, + product: Product, + argv: list[str], + stdin: str | None, + *, + house: str | None, + backend_pin: str | None, + ) -> HandleResult: + chosen, err = self._select_new(product, house=house, backend_pin=backend_pin) + if err is not None or chosen is None: + return err or _gateway("backend_unreachable", "no backend", http=502) + raw = self._forward(chosen, argv, stdin) + if raw is None: + return _gateway("backend_unreachable", chosen.id, http=502) + return _from_backend(raw) + + def _owned( + self, + product: Product, + argv: list[str], + stdin: str | None, + session: str, + *, + house: str | None, + backend_pin: str | None, + ) -> HandleResult: + with self._lock: + row = self.owners.get(session) + if row is None or row.product != product.id or row.backend not in product.backends: + return _gateway("forbidden_session", "not owned by this product", http=403) + backend = self.backends[row.backend] + pinned = self._named_backend(product, house=house, backend_pin=backend_pin) + if isinstance(pinned, HandleResult): + return pinned + if pinned is not None and pinned.id != backend.id: + return _gateway("forbidden_session", "session owned by another backend", http=403) + check = self._require_live(backend, product) + if check is not None: + return check + raw = self._forward(backend, argv, stdin) + if raw is None: + return _gateway("backend_unreachable", backend.id, http=502) + result = _from_backend(raw) + if result.exit_code == 0 and len(argv) >= 2 and argv[0] == "session" and argv[1] == "close": + self._forget(session) + elif result.exit_code == 0: + for sid in _sids(result.stdout, result.stderr): + self._remember_new(sid, product.id, backend.id) + return result + + def _list(self, product: Product, argv: list[str], stdin: str | None) -> HandleResult: + kept: list[str] = [] + others: list[str] = [] + stderr_parts: list[str] = [] + cap: str | None = None + saw_ok = False + gateway_err = False + exit_code = 0 + for backend_id in product.backends: + backend = self.backends[backend_id] + check = self._require_live(backend, product) + if check is not None: + gateway_err = True + stderr_parts.append(check.stderr) + continue + raw = self._forward(backend, argv, stdin) + if raw is None: + gateway_err = True + stderr_parts.append(_gateway("backend_unreachable", backend.id, http=502).stderr) + continue + saw_ok = True + if int(raw.get("exit_code", 1)) != 0 and exit_code == 0: + exit_code = int(raw.get("exit_code", 1)) + stdout = raw.get("stdout") or "" + stderr = raw.get("stderr") or "" + if stderr: + stderr_parts.append(stderr if stderr.endswith("\n") else stderr + "\n") + owned = self._owned_sids(product.id, backend.id) + piece, piece_cap = _filter_sessions(stdout, owned) + kept.extend(piece) + if piece_cap and cap is None: + cap = piece_cap + for line in stdout.splitlines(): + if line.startswith("@SESSION:"): + continue + if _STAT_SESSIONS.match(line): + continue + others.append(line) + lines: list[str] = [] + if cap is not None or kept: + lines.append(f"@STAT: sessions|{len(kept)}/{cap or 0}") + lines.extend(kept) + lines.extend(others) + stdout_out = ("\n".join(lines) + "\n") if lines else "" + if gateway_err: + exit_code = 2 + if not saw_ok and gateway_err: + return HandleResult(502, 2, stdout_out, "".join(stderr_parts)) + return HandleResult(200, exit_code, stdout_out, "".join(stderr_parts)) + + def _select_new( + self, + product: Product, + *, + house: str | None, + backend_pin: str | None, + ) -> tuple[Backend | None, HandleResult | None]: + named = self._named_backend(product, house=house, backend_pin=backend_pin) + if isinstance(named, HandleResult): + return None, named + if named is not None: + check = self._require_live(named, product) + if check is not None: + return None, check + return named, None + mismatch: HandleResult | None = None + unreachable: HandleResult | None = None + for backend_id in product.backends: + backend = self.backends[backend_id] + version = self._version(backend) + if version is None: + unreachable = _gateway("backend_unreachable", backend.id, http=502) + continue + if version != product.pinned_version: + mismatch = _gateway( + "backend_version_mismatch", + f"{backend.id}|got {version}|pin {product.pinned_version}", + http=409, + ) + continue + return backend, None + if mismatch is not None: + return None, mismatch + return None, unreachable or _gateway("backend_unreachable", "none", http=502) + + def _named_backend( + self, + product: Product, + *, + house: str | None, + backend_pin: str | None, + ) -> Backend | HandleResult | None: + if backend_pin: + if backend_pin not in product.backends or backend_pin not in self.backends: + return _gateway("forbidden", "backend is not in this product", http=403) + return self.backends[backend_pin] + if house: + target = product.houses.get(house) + if not target or target not in product.backends: + return _gateway("forbidden", "house is not in this product", http=403) + return self.backends[target] + return None + + def _require_live(self, backend: Backend, product: Product) -> HandleResult | None: + version = self._version(backend) + if version is None: + return _gateway("backend_unreachable", backend.id, http=502) + if version != product.pinned_version: + return _gateway( + "backend_version_mismatch", + f"{backend.id}|got {version}|pin {product.pinned_version}", + http=409, + ) + return None + + def _version(self, backend: Backend) -> str | None: + now = time.monotonic() + with self._lock: + hit = self._ver_cache.get(backend.id) + if ( + hit is not None + and self.version_cache_s > 0 + and (now - hit[0]) < self.version_cache_s + ): + return hit[1] + version: str | None = None + if probe(host=backend.host, port=backend.port): + try: + raw = send_command( + ["version"], + host=backend.host, + port=backend.port, + timeout=self.timeout_s, + ) + except (OSError, json.JSONDecodeError): + raw = None + if isinstance(raw, dict): + for line in (raw.get("stdout") or "").splitlines(): + if line.startswith("@VER: memnet|"): + version = line.split("|", 1)[1].strip() + break + with self._lock: + self._ver_cache[backend.id] = (now, version) + return version + + def _forward( + self, backend: Backend, argv: list[str], stdin: str | None + ) -> dict[str, Any] | None: + try: + raw = send_command( + list(argv), + stdin=stdin, + host=backend.host, + port=backend.port, + timeout=self.timeout_s, + ) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(raw, dict): + return None + err = (raw.get("stderr") or "").strip() + if err == _CLIENT_TIMEOUT and not (raw.get("stdout") or "").strip(): + return None + return raw + + def _product_for_token(self, token: str) -> Product | None: + digest = hashlib.sha256(token.encode()).hexdigest() + found: Product | None = None + for product in self.products.values(): + for cred in product.credentials: + if cred.revoked: + continue + if hmac.compare_digest(cred.sha256, digest): + if found is not None: + return None + found = product + return found + + def _remember(self, sid: str, product_id: str, backend_id: str) -> None: + with self._lock: + current = self.owners.get(sid) + if current is not None and ( + current.product != product_id or current.backend != backend_id + ): + return + self.owners[sid] = OwnerRow(product=product_id, backend=backend_id) + self._save_owners_locked() + + def _remember_new(self, sid: str, product_id: str, backend_id: str) -> None: + with self._lock: + if sid in self.owners: + return + self.owners[sid] = OwnerRow(product=product_id, backend=backend_id) + self._save_owners_locked() + + def _forget(self, sid: str) -> None: + with self._lock: + if sid in self.owners: + del self.owners[sid] + self._save_owners_locked() + + def _owned_sids(self, product_id: str, backend_id: str) -> set[str]: + with self._lock: + return { + sid + for sid, row in self.owners.items() + if row.product == product_id and row.backend == backend_id + } + + def _bump( + self, + product_id: str, + *, + requests: int = 0, + refusals: int = 0, + backend_errors: int = 0, + ) -> None: + with self._lock: + row = self.counts.setdefault(product_id, Counts()) + row.requests += requests + row.gateway_refusals += refusals + row.backend_errors += backend_errors + + def _load_owners(self) -> None: + if not os.path.exists(self.state_path): + return + try: + raw = json.loads(Path(self.state_path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise GatewayConfigError(f"owner state: {exc}") from exc + sessions = raw.get("sessions") if isinstance(raw, dict) else None + if not isinstance(sessions, dict): + raise GatewayConfigError("owner state sessions must be an object") + owners: dict[str, OwnerRow] = {} + for sid, row in sessions.items(): + if not isinstance(sid, str) or not isinstance(row, dict): + raise GatewayConfigError("owner row must map a session id to an object") + product = row.get("product") + backend = row.get("backend") + if product not in self.products or backend not in self.backends: + raise GatewayConfigError("owner row names an unknown product or backend") + if backend not in self.products[product].backends: + raise GatewayConfigError("owner row backend is outside the product") + owners[sid] = OwnerRow(product=str(product), backend=str(backend)) + self.owners = owners + + def _save_owners_locked(self) -> None: + parent = os.path.dirname(self.state_path) + if parent: + os.makedirs(parent, exist_ok=True) + payload = { + "sessions": { + sid: {"product": row.product, "backend": row.backend} + for sid, row in self.owners.items() + } + } + tmp = self.state_path + ".tmp" + Path(tmp).write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + os.replace(tmp, self.state_path) + + +def validate_gateway_bind(host: str) -> None: + """Refuse a public bind unless MEMNET_GATEWAY_ALLOW_PUBLIC=1.""" + if is_loopback_bind_host(host) or _is_tailnet(host): + return + if _allow_public(): + sys.stderr.write( + f"WARNING: memnet product gateway binding to public {host!r} " + "(MEMNET_GATEWAY_ALLOW_PUBLIC=1).\n" + ) + sys.stderr.flush() + return + raise GatewayBindError( + f"refusing public gateway bind {host!r}: loopback or tailnet only " + "(set MEMNET_GATEWAY_ALLOW_PUBLIC=1 to override)" + ) + + +def serve_from_env(*, host: str | None = None, port: int | None = None) -> None: + path = os.environ.get("MEMNET_GATEWAY_CONFIG", "").strip() + if not path: + raise SystemExit(_unconfigured()) + try: + gateway = ProductGateway.load(path, bind_host=host, bind_port=port) + except (GatewayConfigError, GatewayBindError) as exc: + sys.stderr.write(f"@ERR: gateway_bad_request|{exc}\n") + raise SystemExit(2) from exc + gateway.serve_forever() + + +def _unconfigured() -> int: + sys.stderr.write("@ERR: gateway_unconfigured|set MEMNET_GATEWAY_CONFIG\n") + return 2 + + +def _allow_public() -> bool: + raw = os.environ.get("MEMNET_GATEWAY_ALLOW_PUBLIC", "").strip().lower() + return raw in {"1", "true", "yes", "on"} + + +def _is_tailnet(host: str) -> bool: + try: + addr = ipaddress.ip_address(host.strip()) + except ValueError: + return False + if addr.version == 4: + return addr in _CGNAT + return addr in _TAILSCALE_V6 + + +def _gateway(code: str, detail: str, *, http: int) -> HandleResult: + return HandleResult( + http_status=http, + exit_code=2, + stdout="", + stderr=f"@ERR: gateway_{code}|{detail}\n", + ) + + +def _from_backend(raw: dict[str, Any]) -> HandleResult: + return HandleResult( + http_status=200, + exit_code=int(raw.get("exit_code", 1)), + stdout=raw.get("stdout") or "", + stderr=raw.get("stderr") or "", + ) + + +def _bearer(authorization: str | None) -> str | None: + if not authorization: + return None + scheme, _, rest = authorization.strip().partition(" ") + if scheme.lower() != "bearer": + return None + token = rest.strip() + if not token or " " in token: + return None + return token + + +def _hex64(value: str, *, what: str) -> str: + text = value.strip().lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise GatewayConfigError(f"{what} must be a sha256 hex digest") + return text + + +def _parse_backends(raw: Any) -> dict[str, Backend]: + if not isinstance(raw, dict) or not raw: + raise GatewayConfigError("backends must be a non-empty object") + out: dict[str, Backend] = {} + for key, row in raw.items(): + if not isinstance(key, str) or not isinstance(row, dict): + raise GatewayConfigError("backend entry must be an object") + host = str(row.get("host") or "").strip() + if not host: + raise GatewayConfigError(f"backend {key} needs host") + try: + port = int(row["port"]) + except (KeyError, TypeError, ValueError) as exc: + raise GatewayConfigError(f"backend {key} needs port") from exc + if port < 1 or port > 65535: + raise GatewayConfigError(f"backend {key} port out of range") + out[key] = Backend(id=key, host=host, port=port) + return out + + +def _parse_products(raw: Any, backends: dict[str, Backend]) -> dict[str, Product]: + if not isinstance(raw, dict) or not raw: + raise GatewayConfigError("products must be a non-empty object") + out: dict[str, Product] = {} + seen_hashes: dict[str, str] = {} + for key, row in raw.items(): + if not isinstance(key, str) or not isinstance(row, dict): + raise GatewayConfigError("product entry must be an object") + allowed = row.get("backends") + if not isinstance(allowed, list) or not allowed: + raise GatewayConfigError(f"product {key} needs backends") + if not all(isinstance(item, str) and item in backends for item in allowed): + raise GatewayConfigError(f"product {key} names an unknown backend") + houses_raw = row.get("houses") or {} + if not isinstance(houses_raw, dict): + raise GatewayConfigError(f"product {key} houses must be an object") + houses: dict[str, str] = {} + for house, backend_id in houses_raw.items(): + if not isinstance(house, str) or not isinstance(backend_id, str): + raise GatewayConfigError(f"product {key} house must map to a backend id") + if backend_id not in allowed: + raise GatewayConfigError(f"product {key} house {house} is outside its backends") + houses[house] = backend_id + pin = str(row.get("pinned_version") or "").strip() + if not pin: + raise GatewayConfigError(f"product {key} needs pinned_version") + creds_raw = row.get("credentials") + if not isinstance(creds_raw, list) or not creds_raw: + raise GatewayConfigError(f"product {key} needs credentials") + creds: list[Credential] = [] + for cred in creds_raw: + if not isinstance(cred, dict): + raise GatewayConfigError(f"product {key} credential must be an object") + cid = str(cred.get("id") or "").strip() + if not cid: + raise GatewayConfigError(f"product {key} credential needs id") + digest = _hex64(str(cred.get("sha256") or ""), what=f"product {key} credential") + revoked = bool(cred.get("revoked", False)) + if not revoked: + other = seen_hashes.get(digest) + if other is not None: + raise GatewayConfigError("credential hash is shared by two products") + seen_hashes[digest] = key + creds.append(Credential(id=cid, sha256=digest, revoked=revoked)) + out[key] = Product( + id=key, + backends=list(allowed), + houses=houses, + pinned_version=pin, + credentials=creds, + ) + return out + + +def _session_of(payload: dict[str, Any], argv: list[str]) -> str | None: + field_sid = payload.get("session") + if field_sid is not None and not isinstance(field_sid, str): + raise ValueError("session must be a string") + from_field = field_sid.strip() if isinstance(field_sid, str) and field_sid.strip() else None + from_args: str | None = None + if "--session" in argv: + idx = argv.index("--session") + if idx + 1 >= len(argv) or not argv[idx + 1]: + raise ValueError("session flag needs a value") + from_args = argv[idx + 1] + elif len(argv) >= 3 and argv[0] == "session" and argv[1] == "close": + from_args = argv[2] + if from_field and from_args and from_field != from_args: + raise ValueError("session field and args disagree") + return from_field or from_args + + +def _kind(argv: list[str]) -> str: + if argv[0] == "version": + return "probe" + if argv[0] == "session" and len(argv) >= 2: + if argv[1] in _PLACE_CMDS: + return "place" + if argv[1] == "list": + return "list" + if argv[1] == "load" and "--file" in argv and "--session" not in argv: + return "place" + if argv[1] == "expire-status": + return "probe" + return "owned" + + +def _sids(stdout: str, stderr: str) -> list[str]: + found: list[str] = [] + for block in (stdout, stderr): + for line in block.splitlines(): + if not line.startswith("@SESSION:"): + continue + sid = line.split(":", 1)[1].strip().split("|", 1)[0].strip() + if sid and sid not in {"none", "closed"} and sid not in found: + found.append(sid) + return found + + +def _filter_sessions(stdout: str, owned: set[str]) -> tuple[list[str], str | None]: + kept: list[str] = [] + cap: str | None = None + for line in stdout.splitlines(): + match = _STAT_SESSIONS.match(line) + if match: + cap = match.group(2) + continue + if line.startswith("@SESSION:"): + sid = line.split(":", 1)[1].strip().split("|", 1)[0].strip() + if sid in owned: + kept.append(line) + return kept, cap + + +def _handler_factory(gateway: ProductGateway) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: # noqa: N802 + self._respond() + + def do_GET(self) -> None: # noqa: N802 + self._respond() + + def _respond(self) -> None: + length = int(self.headers.get("Content-Length") or "0") + if length > gateway.body_max_bytes: + result = _gateway( + "body_too_large", + f"{length} bytes exceeds {gateway.body_max_bytes}", + http=413, + ) + else: + data = self.rfile.read(length) if length else b"" + result = gateway.handle( + method=self.command, + path=self.path, + authorization=self.headers.get("Authorization"), + body=data, + ) + encoded = json.dumps(result.envelope(), ensure_ascii=False).encode("utf-8") + self.send_response(result.http_status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, fmt: str, *args: Any) -> None: + return + + return Handler diff --git a/parts/memnet-mcp/software/memnet_mcp/server.py b/parts/memnet-mcp/software/memnet_mcp/server.py index 139767f9..3b15be73 100644 --- a/parts/memnet-mcp/software/memnet_mcp/server.py +++ b/parts/memnet-mcp/software/memnet_mcp/server.py @@ -921,9 +921,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument( "--transport", - choices=("stdio", "streamable-http"), + choices=("stdio", "streamable-http", "gateway"), default="stdio", - help="MCP transport (default: stdio; streamable-http is opt-in remote)", + help=( + "MCP transport (default: stdio; streamable-http is opt-in remote; " + "gateway is the product serve router)" + ), ) parser.add_argument( "--host", @@ -974,6 +977,12 @@ def main(argv: list[str] | None = None) -> None: mcp.run(transport="stdio") return + if args.transport == "gateway": + from memnet_mcp.product_gateway import serve_from_env + + serve_from_env(host=args.host, port=args.port) + return + host = args.host if args.host is not None else mcp_http_host() port = args.port if args.port is not None else mcp_http_port() path = args.path if args.path is not None else mcp_http_path() diff --git a/sysml-models/README.md b/sysml-models/README.md index 56a57fd9..445fbe11 100644 --- a/sysml-models/README.md +++ b/sysml-models/README.md @@ -21,12 +21,12 @@ Design authority: rebuilt requirements + ADR-001 (GQL agent wire) + `docs/gramma | File | Package | Role | |------|---------|------| | `models/connections.sysml` | `MemNetConnections` | SharedLlmMemory, SessionHandoff (+ CallerId / SessionBind / SessionCapability), WorkingMemorySlice, SessionImportRequest, optional ImportGuardDecision; ServeUsageLook / ImportGuardArmedLook / HumanUsagePage (ops look); application `CompanyAnalyticalSsot` / `HostSearchBridge` / `DeviceMemNetFleet` / `TipMemNetAccess` / `MemNetLanMcpCluster` / `MemNetTwoMovesContrast` | -| `models/requirements.sysml` | `MemNetRequirements` | MN-REQ-00…13 (01.7/01.8, 06.4, **06.5** human usage look, **06.6** device services / one droplet MemNet MCP tip/ops, product face sysmledge, **06.7** SSOT → code allocate, **06.8** tip MemNet access portal, **06.9** ClusterRoute LAN MCP front invent #191, **06.10** SliceHandCarry invent #47 cousin, 12.9–12.13, 13.1 Recall/Commit; 02.9 cousin store-key; 04.8 cue \|Q\|>1; 04.9 empty-q outline) | +| `models/requirements.sysml` | `MemNetRequirements` | MN-REQ-00…13 (01.7/01.8, 06.4, **06.5** human usage look, **06.6** device services / one droplet MemNet MCP tip/ops, product face sysmledge, **06.7** SSOT → code allocate, **06.8** tip MemNet access portal, **06.9** ClusterRoute LAN MCP front invent #191, **06.10** SliceHandCarry invent #47 cousin, **06.12** product gateway on memnet-mcp, 12.9–12.13, 13.1 Recall/Commit; 02.9 cousin store-key; 04.8 cue \|Q\|>1; 04.9 empty-q outline) | | `models/cousins.sysml` | `MemNetCousinContrast` | TARGET vs eight cousin pointing/identity designs (not a product switch; SysMLEdge is a distinct pin_map; overlay family `SysMLEdgePrj-*`; this engine repo and `modelbasedPrj-*` are repo-based — MUST NOT use SysMLEdge as model SSOT; `SysMLEdgePrj-*` bound desk is working model SSOT) | -| `models/deploy.sysml` | `MemNet` | Nested parts; `RecallCommit` two-operator cut; Multitask spine; `MemNetUsageDashboard` / `MemNetOpsFleet` / `TipMemNetAccessPortal` / `MemNetLanMcpFront` / `MemNetTwoMoves` outside `MemNetSystem` | +| `models/deploy.sysml` | `MemNet` | Nested parts; `RecallCommit` two-operator cut; Multitask spine; `MemNetUsageDashboard` / `MemNetOpsFleet` / `TipMemNetAccessPortal` / `MemNetLanMcpFront` / `MemNetProductGateway` / `MemNetTwoMoves` outside `MemNetSystem` | | `models/implementation.sysml` | `MemNetImplementation` | `SoftwareAllocate` logical → live modules; one Hatch wheel many hosts; sysmledge not in wheel | | `models/behaviour.sysml` | `MemNetBehaviour` | HandoffById, SessionImportReceive, Multitask async, landed-client hydrate/flush | -| `models/verify.sysml` | `MemNetVerification` | MN-VER-12-G00 + S01…S14; MN-VER-04-S01…S05; MN-VER-09-S01; MN-VER-13-S01; MN-VER-06-S01…S08 (S06 storage roles; S07 ClusterRoute LAN MCP front; S08 two named moves); MN-VER-01-S03 | +| `models/verify.sysml` | `MemNetVerification` | MN-VER-12-G00 + S01…S14; MN-VER-04-S01…S05; MN-VER-09-S01; MN-VER-13-S01; MN-VER-06-S01…S10 (S06 storage roles; S07 ClusterRoute LAN MCP front; S08 two named moves; S09 admin usage; S10 product gateway); MN-VER-01-S03 | | `models/root.sysml` | `ProjectMemNet` | Root imports (load last). MUST NOT import `MemNetArchive` | | `models/archive.sysml` | `MemNetArchive` | ARCHIVE shelf (leftover_* / TierACodec / LegacyPipe* / retired Neo4j). **Off** `config.yaml` load | @@ -106,6 +106,7 @@ Two shelves (detail + principles: [outputs/README.md](outputs/README.md)). **Pro | Device fleet (one MemNet MCP at droplet, tip≠face; not #47) | [outputs/device-fleet-one-mcp-case-study.md](outputs/device-fleet-one-mcp-case-study.md) | | Tip MemNet access portal (keyed Bearer; portal sidecar) | [outputs/tip-memnet-access-portal-case-study.md](outputs/tip-memnet-access-portal-case-study.md) | | LAN MCP front (ClusterRoute; one catalogue, N LAN serves; inventOnly #191) | [outputs/lan-mcp-front-case-study.md](outputs/lan-mcp-front-case-study.md) | +| Product gateway (memnet-mcp route to one owning serve; MN-REQ-06.12) | [outputs/product-gateway-case-study.md](outputs/product-gateway-case-study.md) | | ClusterRoute vs SliceHandCarry (two named moves; inventOnly #191 / #47 cousin) | [outputs/cluster-route-vs-slice-hand-carry-case-study.md](outputs/cluster-route-vs-slice-hand-carry-case-study.md) | | SSOT → code allocate (one Hatch wheel, many hosts) | [outputs/ssot-to-code-allocate-case-study.md](outputs/ssot-to-code-allocate-case-study.md) | | SSOT → code tracker map | [outputs/ssot-to-code-allocate-map.md](outputs/ssot-to-code-allocate-map.md) | diff --git a/sysml-models/config.yaml b/sysml-models/config.yaml index d107c004..49ea5861 100644 --- a/sysml-models/config.yaml +++ b/sysml-models/config.yaml @@ -19,7 +19,9 @@ model_files: # TipMemNetAccessPortal nests in deploy.sysml (outside MemNetSystem; # same load as MemNetUsageDashboard / MemNetOpsFleet). No extra file. # MemNetLanMcpFront nests in deploy.sysml (outside MemNetSystem; - # same load; invent-only #191; ClusterRoute). No extra file. + # same load; invent-only #191 catalogue; ClusterRoute). + # MemNetProductGateway nests on that front (MN-REQ-06.12; shipped). + # No extra file. # MemNetTwoMoves / SliceHandCarry nests in deploy.sysml # (outside MemNetSystem; invent-only #47 cousin). No extra file. - implementation.sysml diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index 6caf59a0..2d1598cc 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -97,7 +97,10 @@ is not sole routing). One owner per session; no cross-backend graph merge. pin_map / find SHALL NOT span backends. Distinct from SliceHandCarry (MN-REQ-06.10 / #47 cousin). tip≠face; - inventOnly; no SemVer. + inventOnly; no SemVer. Nested MemNetProductGateway (MN-REQ-06.12) + is the shipped memnet-mcp cut: per-product credentials, one + owning serve per session, exact serve-envelope pass-through. + The #191 catalogue union and SnapshotHandCarry stay inventOnly. APPLICATION / OPS TWO MOVES (not in MemNetSystem): MemNetTwoMoves — one nest ClusterRoute vs SliceHandCarry so agents do not conflate routing (where the session lives) with an explicit copy into @@ -3453,6 +3456,58 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_9_LanMcpFrontSeveralServes; } + part def MemNetProductGateway { + doc /* + Shipped capability of memnet-mcp (MN-REQ-06.12), nested + on MemNetLanMcpFront. Not a separate droplet shim. + Registry maps backend id to host:port (no host literal + in code). Credential is per product, sha256 only, scoped + to that product's backends, rotatable and revocable. + Does not accept mn_tip_ keys. One owner per session. + No silent cross-backend move. Failover only for a new + session when house and backend are unnamed. Args and + stdin pass through. Engine control lines pass through + except the session-list membership filter. No caps below + the backend except the declared body ceiling. Version + pin per product. List is filtered. Foreign session is + gateway_forbidden_session. Bind is loopback or tailnet + unless MEMNET_GATEWAY_ALLOW_PUBLIC=1. Admin counts omit + content and session ids. Parent catalogue stays + inventOnly. + */ + attribute insideMemNetSystem : Boolean = false; + attribute tipIsFace : Boolean = false; + attribute isSysmlEdgeProduct : Boolean = false; + attribute inventOnly : Boolean = false; + attribute implemented : Boolean = true; + attribute codeApproved : Boolean = true; + attribute noSemVerBump : Boolean = true; + attribute reusesTipBearer : Boolean = false; + attribute hashedCredentials : Boolean = true; + attribute credentialScopedToProductBackends : Boolean = true; + attribute rotateAndRevoke : Boolean = true; + attribute oneOwnerPerSession : Boolean = true; + attribute silentCrossBackendMove : Boolean = false; + attribute failoverOnlyForNewSession : Boolean = true; + attribute passThroughGql : Boolean = true; + attribute passThroughStdin : Boolean = true; + attribute verbatimEngineControlLines : Boolean = true; + attribute gatewayCapsBelowBackend : Boolean = false; + attribute bodyCeilingDeclared : Boolean = true; + attribute versionPinPerProduct : Boolean = true; + attribute listFilteredByProduct : Boolean = true; + attribute foreignSessionGatewayErr : Boolean = true; + attribute publicBindDefault : Boolean = false; + attribute loopbackOrTailnetOnly : Boolean = true; + attribute adminCountsOmitContent : Boolean = true; + attribute adminCountsOmitSessionId : Boolean = true; + attribute singleBackendUnchanged : Boolean = true; + attribute hardcodedHost : Boolean = false; + + satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_12_ProductGateway; + satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_9_LanMcpFrontSeveralServes; + } + part def MemNetLanMcpFront { doc /* OPS nest outside MemNetSystem (MN-REQ-06.9 / #191). One @@ -3469,13 +3524,16 @@ package MemNet { overlay is optional ops. One serve already hosts many sessions (singleServeHostsManySessions); this nest makes the complexity-versus-capacity trade-off visible without - approving engine code. + approving that catalogue. Nested productGateway is the + shipped memnet-mcp cut (MN-REQ-06.12; implemented=true). + MCP tool union and SnapshotHandCarry stay inventOnly. */ part clusterRoute : ClusterRoute; part registry : SessionOwnerRegistry; part backends : ServeBackend[1..*]; part auth : FrontBackendAuth; part handCarry : SnapshotHandCarry; + part productGateway : MemNetProductGateway; port routeOut : RoutedMcpCallOutPort; port pinOut : SessionOwnerRecordOutPort; port ownerIn : SessionOwnerRecordInPort; diff --git a/sysml-models/models/implementation.sysml b/sysml-models/models/implementation.sysml index 5003ea0b..b34ba787 100644 --- a/sysml-models/models/implementation.sysml +++ b/sysml-models/models/implementation.sysml @@ -4,10 +4,11 @@ One Hatch wheel (memnet-llm) for many hosts (droplet + devices). MemNet MCP module = tip/ops (tip≠face). Cousin sysmledge is NOT allocated here (thisRepoHasProductFace=false; mustNotInventUploadBind). - Not N-server federation (#47). LAN MCP front (#191) is invent-only - in deploy.sysml (MemNetLanMcpFront / ClusterRoute); SliceHandCarry - (#47 cousin / MN-REQ-06.10) is invent-only; MUST NOT allocate a - Python path from this package. + Not N-server federation (#47). LAN MCP front catalogue (#191) stays + invent-only (MemNetLanMcpFront / ClusterRoute / SnapshotHandCarry). + SliceHandCarry (#47 cousin / MN-REQ-06.10) is invent-only; MUST NOT + allocate a Python path for that copy. MemNetProductGateway + (MN-REQ-06.12) is the shipped memnet-mcp capability and IS allocated. */ package MemNetImplementation { private import ScalarValues::*; @@ -242,6 +243,13 @@ package MemNetImplementation { attribute path : String = "parts/memnet-mcp/software/memnet_mcp/http_transport.py"; attribute pyName : String = "memnet_mcp.http_transport"; } + part def ProductGatewayMod { + doc /* Product gateway on memnet-mcp (MN-REQ-06.12). Not mn_tip_. */ + attribute path : String = "parts/memnet-mcp/software/memnet_mcp/product_gateway.py"; + attribute pyName : String = "memnet_mcp.product_gateway"; + attribute tipIsFace : Boolean = false; + attribute reusesTipBearer : Boolean = false; + } // ----- Cousin product face: NOT in this repo ----- @@ -293,6 +301,7 @@ package MemNetImplementation { part mcpBridgeMod : McpServeBridgeMod; part lawSeedMod : LawSeedHelperMod; part mcpHttpMod : McpHttpTransportMod; + part productGatewayMod : ProductGatewayMod; part sysmlEdgeAbsent : CousinSysMLEdgeNotInRepo; part tracker : ImplementationTracker; @@ -466,6 +475,11 @@ package MemNetImplementation { end logical ::> memNetSystem.mcp.seed; end code ::> lawSeedMod; } + allocation productGatewayToMod : SoftwareAllocate { + doc /* Shipped product gateway. Parent #191 catalogue stays invent-only. */ + end logical ::> lanMcpFront.productGateway; + end code ::> productGatewayMod; + } // CousinSysMLEdge has no SoftwareAllocate in this package. } diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index cc972b89..bafc9edd 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -27,6 +27,7 @@ MN-REQ-06.9 LAN MCP front invent (#191): ClusterRoute — where the session lives; one MCP catalogue, N LAN serves; cousin of #47; inventOnly MN-REQ-06.10 SliceHandCarry invent (#47 cousin): bounded WorkingMemorySlice copy into another session; not a live hop; inventOnly MN-REQ-06.11 Admin-only serve usage report (counts/caps; opaque session alias; not agent MCP) + MN-REQ-06.12 Product gateway on memnet-mcp: per-product route to owning serve (not mn_tip_) MN-REQ-11.17 Catalog Snap / model Snap — session strata (0.15) MN-REQ-11.17.1 Cross-cut satisfy locators on the catalog; SysMLEdge pin_map is a different tool (not this catalog) @@ -937,6 +938,63 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-06.11"; } + + requirement def MN_REQ_06_12_ProductGateway { + doc /* + SHALL provide MemNetProductGateway as a capability of + memnet-mcp (MemNetLanMcpFront), not a separate droplet + shim. Online products (Endleaf now; Atelier and + SysMLEdge later) SHALL reach a memnet serve only + through this gateway. tip≠face. SHALL NOT nest under + MemNetSystem. SHALL NOT bump SemVer for this cut. + When MEMNET_GATEWAY_CONFIG is unset, memnet-mcp SHALL + keep today's single backend: in-process by default, + or one MEMNET_SERVE_HOST and MEMNET_SERVE_PORT when + MEMNET_MCP_TRANSPORT=tcp. + Registry: a config file SHALL map each backend id to + a tailnet host and port. No host literal in code. + Route by product (the credential), optional house, + and session. SessionOwnerRegistry row SHALL be the + one owner. SHALL NOT move a live session onto another + backend. Failover SHALL apply only when placing a + new session (session open, or session load from a + file) and the request names neither house nor + backend. A named house or backend that is down SHALL + refuse gateway_backend_unreachable. Owner down on a + later call SHALL refuse the same way. + Auth: one credential set per product, stored as sha256 + only, scoped to that product's backends, rotatable by + adding a hash and revocable by flag. SHALL NOT accept + mn_tip_ portal keys or MEMNET_MCP_HTTP_TOKEN as this + credential (MN-REQ-06.8 is a different hop). + Pass-through: args and stdin SHALL be forwarded + unchanged. Engine @ERR, @WRN, @STAT and exit_code + SHALL return as the backend wrote them, except the + session-list membership filter (filtered n in + @STAT: sessions|n/max). The gateway SHALL NOT enforce + MAX_ROWS, mutate batch lines, or value bytes below + the backend. The only gateway size cap is the + request-body ceiling (default 4 MiB, contract). + Each product SHALL pin a backend version. Mismatch + SHALL refuse gateway_backend_version_mismatch and + SHALL NOT forward. + Session open, load, save, close and list SHALL stay + inside the product namespace. List SHALL omit another + product's sessions. A foreign or unknown session + SHALL refuse gateway_forbidden_session. + Listener SHALL bind loopback or tailnet (100.64.0.0/10 + or Tailscale IPv6) by default. A public bind SHALL + be refused unless MEMNET_GATEWAY_ALLOW_PUBLIC=1. + Gateway refusals SHALL be @ERR: gateway_|… + distinct from engine @ERR. Admin counts SHALL be per + product and SHALL omit graph content and session ids. + The #191 catalogue (MCP tool union across backends, + SnapshotHandCarry) stays inventOnly on + MemNetLanMcpFront. This leaf is the shipped cut + (implemented=true). + */ + attribute requirementId : String = "MN-REQ-06.12"; + } } // ----- MCP agent boundary ----- @@ -1955,6 +2013,8 @@ package MemNetRequirements { : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_10_SliceHandCarryAcrossServes; requirement adminServeUsageReportReq : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_11_AdminServeUsageReport; + requirement productGatewayReq + : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_12_ProductGateway; requirement mcpAgentBoundaryReq : MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary; @@ -2188,6 +2248,7 @@ package MemNetRequirements { end #derive ::> lanMcpFrontSeveralServesReq; end #derive ::> sliceHandCarryAcrossServesReq; end #derive ::> adminServeUsageReportReq; + end #derive ::> productGatewayReq; } #derivation connection deriveMcpLeaves { diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 08a67b90..1688bc59 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -46,6 +46,10 @@ not a live hop; import_slice same-serve only; inventOnly. Admin serve usage report (MN-VER-06-S09): AdminUsageReport on TcpServeBridge; admin credential; opaque alias; not agent MCP. + Product gateway (MN-VER-06-S10): MemNetProductGateway on + memnet-mcp; per-product sha256 credential; one owning serve; + no mn_tip_ reuse; pass-through; implemented. Parent #191 + catalogue stays inventOnly. Optional id nickname TARGET (MN-VER-02-S01): GraphElement identity; CREATE () legal; HiddenStoreHandle off-wire; 0.9 leftover_by_id store. TARGET command list cue/pattern only (MN-VER-07-S01): no required-id @@ -1573,6 +1577,56 @@ package MemNetVerification { } } + verification def MN_VER_06_S10_ProductGateway { + doc /* + MN-REQ-06.12 — MemNetProductGateway on memnet-mcp. + Outside MemNetSystem. tip≠face. Hashed per-product + credentials, not mn_tip_. One owner. No silent move. + Pass-through of GQL and stdin. No caps below the + backend. Body ceiling declared. Version pin. Filtered + list. Loopback or tailnet by default. Admin counts omit + content and session ids. Single-backend memnet-mcp + unchanged when no registry is configured. No SemVer bump. + */ + attribute verificationId : String = "MN-VER-06-S10"; + + subject gateway : MemNetProductGateway; + + objective productGateway { + verify productGatewayReq; + require constraint { + gateway.insideMemNetSystem == false + and gateway.tipIsFace == false + and gateway.isSysmlEdgeProduct == false + and gateway.inventOnly == false + and gateway.implemented == true + and gateway.codeApproved == true + and gateway.noSemVerBump == true + and gateway.reusesTipBearer == false + and gateway.hashedCredentials == true + and gateway.credentialScopedToProductBackends == true + and gateway.rotateAndRevoke == true + and gateway.oneOwnerPerSession == true + and gateway.silentCrossBackendMove == false + and gateway.failoverOnlyForNewSession == true + and gateway.passThroughGql == true + and gateway.passThroughStdin == true + and gateway.verbatimEngineControlLines == true + and gateway.gatewayCapsBelowBackend == false + and gateway.bodyCeilingDeclared == true + and gateway.versionPinPerProduct == true + and gateway.listFilteredByProduct == true + and gateway.foreignSessionGatewayErr == true + and gateway.publicBindDefault == false + and gateway.loopbackOrTailnetOnly == true + and gateway.adminCountsOmitContent == true + and gateway.adminCountsOmitSessionId == true + and gateway.singleBackendUnchanged == true + and gateway.hardcodedHost == false + } + } + } + verification def MN_VER_13_S01_RecallCommitTwoOperators { doc /* MN-REQ-13.1 — two-operator cut (math skeleton; leftover goldfish). diff --git a/sysml-models/outputs/README.md b/sysml-models/outputs/README.md index 4cadca94..92c18bce 100644 --- a/sysml-models/outputs/README.md +++ b/sysml-models/outputs/README.md @@ -34,6 +34,7 @@ Keep product-canon and GQL application studies. Do not restore leftover `NEW` mi | [device-fleet-one-mcp-case-study.md](device-fleet-one-mcp-case-study.md) | Device MemNet services; one MemNet MCP (tip/ops) at the droplet; product face is sysmledge | MN-REQ-06.6; MN-VER-06-S03; tip≠face; not #47 | | [tip-memnet-access-portal-case-study.md](tip-memnet-access-portal-case-study.md) | Admin invite, Google login, Bearer for keyed tip MCP; unauthenticated WWW MemNet refused | MN-REQ-06.8; MN-VER-06-S05; tip≠face; portal sidecar | | [lan-mcp-front-case-study.md](lan-mcp-front-case-study.md) | One MCP catalogue over N LAN serves; ClusterRoute; registry owner; not shipped | MN-REQ-06.9; MN-VER-06-S07; tip≠face; inventOnly #191 | +| [product-gateway-case-study.md](product-gateway-case-study.md) | memnet-mcp routes a product to one owning serve; parent catalogue stays inventOnly | MN-REQ-06.12; MN-VER-06-S10; tip≠face; not mn_tip_ | | [cluster-route-vs-slice-hand-carry-case-study.md](cluster-route-vs-slice-hand-carry-case-study.md) | Two named moves: ClusterRoute vs SliceHandCarry; not shipped | MN-REQ-06.9 + 06.10; MN-VER-06-S08; tip≠face; inventOnly #191 / #47 cousin | | [ssot-to-code-allocate-case-study.md](ssot-to-code-allocate-case-study.md) | SSOT parts → live Python modules; one Hatch wheel many hosts; sysmledge not in wheel | MN-REQ-06.7; MN-VER-06-S04; SoftwareAllocate | | [ssot-to-code-allocate-map.md](ssot-to-code-allocate-map.md) | Implementation tracker ledger (every allocate row → path) | `ImplementationTracker`; missing path fails CI | diff --git a/sysml-models/outputs/product-gateway-case-study.md b/sysml-models/outputs/product-gateway-case-study.md new file mode 100644 index 00000000..df7dd7c3 --- /dev/null +++ b/sysml-models/outputs/product-gateway-case-study.md @@ -0,0 +1,27 @@ +# Case study: product gateway on memnet-mcp (MN-REQ-06.12) + +**Shelf:** product canon — shipped memnet-mcp capability (parent catalogue stays inventOnly #191) + +Online products reach a `memnet serve` through `memnet-mcp --transport gateway`. Endleaf now. Atelier and SysMLEdge later, each with their own credential and backend list. The #191 MCP tool union and SnapshotHandCarry stay invent-only on `MemNetLanMcpFront`. Lock: **tip≠face**. Product credentials are not `mn_tip_` keys. No SemVer. Wire: [`docs/operations/product-gateway-contract.md`](../../docs/operations/product-gateway-contract.md). + +**Wire:** the gateway forwards argv and stdin. It does not merge graphs. A session has one owning serve. Chat is never SSOT. + +## 1. Purpose + +Give each product a scoped credential and a backend list, place each new session on one live serve that matches the version pin, and refuse a foreign session instead of moving it. + +## 2. Model locus + +| Concern | SysML | +|---------|-------| +| Requirement | **MN-REQ-06.12** (`productGatewayReq`) | +| Verify | **MN-VER-06-S10** | +| Part | `MemNetProductGateway` nested on `MemNetLanMcpFront`, **outside** `MemNetSystem` | +| Code | `productGatewayMod` → `parts/memnet-mcp/software/memnet_mcp/product_gateway.py` | + +```text +MemNetLanMcpFront // OUTSIDE — inventOnly #191 (unchanged) +└── MemNetProductGateway // shipped child; implemented=true +``` + +Parent attributes stay `inventOnly=true`, `implemented=false`, `codeApproved=false`. The child is `reusesTipBearer=false`, `oneOwnerPerSession=true`, `silentCrossBackendMove=false`, `failoverOnlyForNewSession=true`, `publicBindDefault=false`. diff --git a/sysml-models/outputs/product-nest-one-page.md b/sysml-models/outputs/product-nest-one-page.md index 6704da3d..79727853 100644 --- a/sysml-models/outputs/product-nest-one-page.md +++ b/sysml-models/outputs/product-nest-one-page.md @@ -29,6 +29,7 @@ SERVE ADMIN LOOK AdminUsageReport on TcpServeBridge — admin JSON; opaque ali OPS FLEET MemNetOpsFleet — device MemNet services; one MemNet MCP at droplet (tip/ops; tip≠face; product face is sysmledge) OPS ACCESS TipMemNetAccessPortal — Szu-Wei invite, Google login, Bearer for keyed tip MCP at droplet WWW (tip≠face; not sysmledge; portal sidecar; look-only service/client status) OPS LAN FRONT MemNetLanMcpFront — ClusterRoute: one MCP catalogue over N LAN serves (inventOnly #191; tip≠face; not shipped) +OPS PRODUCT GATEWAY MemNetProductGateway — memnet-mcp capability (MN-REQ-06.12; implemented; tip≠face; not mn_tip_) OPS TWO MOVES MemNetTwoMoves — ClusterRoute vs SliceHandCarry (inventOnly #191 / #47 cousin; explicit copy ≠ live hop; not shipped) IMPLEMENTATION MemNetImplementation — SoftwareAllocate SSOT → live modules; tracker ledger; one Hatch wheel many hosts; sysmledge not in wheel ``` @@ -53,6 +54,7 @@ IMPLEMENTATION MemNetImplementation — SoftwareAllocate SSOT → live modul - Unparking engine usage-dashboard HTTP from the portal status look (`unparksUsageDashboard=false`; dashboard `httpImplemented=false`) - N-server federation (`nServerFederation=false`; #47) - LAN MCP front as shipped code (`MemNetLanMcpFront.inventOnly=true`; `implemented=false`; `codeApproved=false`; #191) +- Treating the nested product gateway as the #191 catalogue (`MemNetProductGateway` is the shipped memnet-mcp cut; parent stays inventOnly) - SliceHandCarry as a live hop or cross-host `import_slice` (`SliceHandCarry.isLiveHop=false`; `importSliceAcrossHosts=false`; `importSliceFromUrl=false`; #47 cousin) - Silent hash as sole session routing (`silentHashRouting=false`; registry entry preferred) - `pin_map` / `find` spanning backends (`pinMapSpansBackends=false`) @@ -67,4 +69,4 @@ Honesty that leftovers exist lives on the **ARCHIVE** shelf, not on `ProjectMemN Session TTL drops **RAM**. Explicit `session_save` and expire-save write **one** file blob. Expire-save is **off** unless `MEMNET_SAVE_ON_EXPIRE`. Disk file stays until the user deletes it; `session_load` restores RAM (`session_load(session=)` resolves the expire-dir snap when the caller already holds the sid; `loadByKnownSid`). DurableBuffer is a cabinet ego slice, not this file (`MN-VER-01-S03`, `MN-VER-06-S06`). Neo4j is archived (#187). -ARCHIVE leftover fog remains **off** `ProjectMemNet` load (`leftoverFogNested=false`, `leftoverArchiveOffLoad=true`). OPS `MemNetUsageDashboard` remains look-only (`httpImplemented=false`, `tipIsFace=false`, `agentWire=false`). SERVE `AdminUsageReport` remains on `TcpServeBridge` (`onAgentMcp=false`, `emitsRealSessionId=false`, `peekOnly=true`, `mcpProductLabelOpen=true`). OPS `MemNetOpsFleet` remains outside `MemNetSystem` (`mcpCount=1` = one MemNet MCP at droplet, `nServerFederation=false`, `productInventFace=sysmledge`, `tipIsFace=false`). OPS `TipMemNetAccessPortal` remains outside `MemNetSystem` (`tipIsFace=false`, `keyedWwwMcp=true`, `unauthenticatedWwwMcp=false`, `isSysmlEdgeProduct=false`, `inventOnly=false`, `portalWebImplemented=true`, `sellsInvent2Green=false`, `statusLookOnly=true`, `unparksUsageDashboard=false`). OPS `MemNetLanMcpFront` remains outside `MemNetSystem` (`inventOnly=true`, `implemented=false`, `codeApproved=false`, `nServerPeerHandoff=false`, `pinMapSpansBackends=false`, `worthBuildingVisible=true`; #191). OPS `MemNetTwoMoves` remains outside `MemNetSystem` (`inventOnly=true`, `implemented=false`, `clusterIsWhereSessionLives=true`, `sliceIsExplicitCopy=true`, `liveHop=false`, `importSliceAcrossHosts=false`; #191 / #47 cousin). IMPLEMENTATION `MemNetLlmWheel` remains one Hatch package for many hosts (`oneWheelManyHosts=true`, `sysmlEdgeInWheel=false`). IMPLEMENTATION tracker (`ImplementationTracker`) watches allocate rows (`missingPathFailsCi`; `sysmlEdgeTracked=false`). +ARCHIVE leftover fog remains **off** `ProjectMemNet` load (`leftoverFogNested=false`, `leftoverArchiveOffLoad=true`). OPS `MemNetUsageDashboard` remains look-only (`httpImplemented=false`, `tipIsFace=false`, `agentWire=false`). SERVE `AdminUsageReport` remains on `TcpServeBridge` (`onAgentMcp=false`, `emitsRealSessionId=false`, `peekOnly=true`, `mcpProductLabelOpen=true`). OPS `MemNetOpsFleet` remains outside `MemNetSystem` (`mcpCount=1` = one MemNet MCP at droplet, `nServerFederation=false`, `productInventFace=sysmledge`, `tipIsFace=false`). OPS `TipMemNetAccessPortal` remains outside `MemNetSystem` (`tipIsFace=false`, `keyedWwwMcp=true`, `unauthenticatedWwwMcp=false`, `isSysmlEdgeProduct=false`, `inventOnly=false`, `portalWebImplemented=true`, `sellsInvent2Green=false`, `statusLookOnly=true`, `unparksUsageDashboard=false`). OPS `MemNetLanMcpFront` remains outside `MemNetSystem` (`inventOnly=true`, `implemented=false`, `codeApproved=false`, `nServerPeerHandoff=false`, `pinMapSpansBackends=false`, `worthBuildingVisible=true`; #191). Nested `MemNetProductGateway` is the shipped memnet-mcp cut (`implemented=true`, `reusesTipBearer=false`; MN-REQ-06.12) while the parent catalogue stays inventOnly. OPS `MemNetTwoMoves` remains outside `MemNetSystem` (`inventOnly=true`, `implemented=false`, `clusterIsWhereSessionLives=true`, `sliceIsExplicitCopy=true`, `liveHop=false`, `importSliceAcrossHosts=false`; #191 / #47 cousin). IMPLEMENTATION `MemNetLlmWheel` remains one Hatch package for many hosts (`oneWheelManyHosts=true`, `sysmlEdgeInWheel=false`). IMPLEMENTATION tracker (`ImplementationTracker`) watches allocate rows (`missingPathFailsCi`; `sysmlEdgeTracked=false`). diff --git a/sysml-models/outputs/ssot-to-code-allocate-map.md b/sysml-models/outputs/ssot-to-code-allocate-map.md index 6e9c6453..66e4c53d 100644 --- a/sysml-models/outputs/ssot-to-code-allocate-map.md +++ b/sysml-models/outputs/ssot-to-code-allocate-map.md @@ -49,6 +49,7 @@ | mcpBridgeToMod | `memNetSystem.mcp.bridge` | `mcpBridgeMod` | `parts/memnet-mcp/software/memnet_mcp/serve_bridge.py` | `memnet_mcp.serve_bridge` | true | | mcpHttpToMod | `memNetSystem.multitask.sharedStore` | `mcpHttpMod` | `parts/memnet-mcp/software/memnet_mcp/http_transport.py` | `memnet_mcp.http_transport` | true | | lawSeedToMod | `memNetSystem.mcp.seed` | `lawSeedMod` | `parts/memnet-mcp/software/memnet_mcp/law_seed_helper.py` | `memnet_mcp.law_seed_helper` | true | +| productGatewayToMod | `lanMcpFront.productGateway` | `productGatewayMod` | `parts/memnet-mcp/software/memnet_mcp/product_gateway.py` | `memnet_mcp.product_gateway` | true | Ellipsis `…` shortens `memNetSystem.core.transport.inProcess.memory.sessions` (and RecallCommit under that). Full qnames live on the `end logical` lines in `implementation.sysml`. diff --git a/tests/test_product_gateway.py b/tests/test_product_gateway.py new file mode 100644 index 00000000..d3d21a9b --- /dev/null +++ b/tests/test_product_gateway.py @@ -0,0 +1,495 @@ +"""Product gateway: two real serve processes, plus routing unit checks.""" + +from __future__ import annotations + +import hashlib +import json +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +from memnet.serve import probe, send_command +from memnet_mcp.client import _transport, run_memnet +from memnet_mcp.product_gateway import ( + GatewayBindError, + ProductGateway, + validate_gateway_bind, +) +from memnet_mcp.server import _parse_args + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = ROOT / "parts" / "common" / "memnet" / "memnet" / "examples" / "schema.example.txt" +ENDLEAF = "endleaf-secret" +ATELIER = "atelier-secret" +ADMIN = "admin-secret" + + +def _sha(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _config( + path: Path, + state: Path, + port_a: int, + port_b: int, + *, + pin: str = "0.19.18", + body_max: int = 4 * 1024 * 1024, +) -> None: + data = { + "bind": "127.0.0.1", + "port": 0, + "path": "/gateway", + "body_max_bytes": body_max, + "state_path": str(state), + "version_cache_s": 0, + "backend_timeout_s": 10, + "admin": {"sha256": _sha(ADMIN)}, + "backends": { + "a": {"host": "127.0.0.1", "port": port_a}, + "b": {"host": "127.0.0.1", "port": port_b}, + }, + "products": { + "endleaf": { + "backends": ["a", "b"], + "houses": {"syson": "a"}, + "pinned_version": pin, + "credentials": [ + {"id": "endleaf-1", "sha256": _sha(ENDLEAF), "revoked": False}, + {"id": "endleaf-old", "sha256": _sha("endleaf-old"), "revoked": True}, + ], + }, + "atelier": { + "backends": ["b"], + "houses": {}, + "pinned_version": "0.19.18", + "credentials": [ + {"id": "atelier-1", "sha256": _sha(ATELIER), "revoked": False}, + ], + }, + }, + } + path.write_text(json.dumps(data), encoding="utf-8") + + +def _start_serve(port: int, log_path: Path) -> subprocess.Popen[bytes]: + env = os.environ.copy() + for key in ( + "MEMNET_TEST_INLINE", + "MEMNET_SERVE_INTERNAL", + "MEMNET_SESSION", + "MEMNET_GATEWAY_CONFIG", + "MEMNET_GATEWAY_ALLOW_PUBLIC", + ): + env.pop(key, None) + env["MEMNET_SERVE_HOST"] = "127.0.0.1" + env["MEMNET_SERVE_PORT"] = str(port) + log = log_path.open("w", encoding="utf-8") + proc = subprocess.Popen( + [sys.executable, "-m", "memnet", "serve", "--host", "127.0.0.1", "--port", str(port)], + env=env, + cwd=str(ROOT), + stdout=log, + stderr=subprocess.STDOUT, + ) + log.close() + deadline = time.time() + 20 + while time.time() < deadline: + if proc.poll() is not None: + raise AssertionError(log_path.read_text(encoding="utf-8")) + if probe(host="127.0.0.1", port=port): + return proc + time.sleep(0.05) + proc.terminate() + raise AssertionError("memnet serve did not start:\n" + log_path.read_text(encoding="utf-8")) + + +def _stop(proc: subprocess.Popen[bytes] | None) -> None: + if proc is None or proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def _post(url: str, token: str | None, payload: dict) -> tuple[int, dict]: + data = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, data=data, method="POST", headers=headers) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + return resp.status, json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def _sid(body: dict) -> str: + for line in (body.get("stdout") or "").splitlines(): + if line.startswith("@SESSION:"): + return line.split(":", 1)[1].strip().split("|", 1)[0].strip() + raise AssertionError(body) + + +def _live_sids(port: int) -> set[str]: + raw = send_command(["session", "list"], host="127.0.0.1", port=port, timeout=5) + found = set() + for line in (raw.get("stdout") or "").splitlines(): + if line.startswith("@SESSION:"): + found.add(line.split(":", 1)[1].strip().split("|", 1)[0].strip()) + return found + + +def test_bind_is_loopback_or_tailnet(monkeypatch: pytest.MonkeyPatch) -> None: + validate_gateway_bind("127.0.0.1") + validate_gateway_bind("localhost") + validate_gateway_bind("100.118.79.40") + with pytest.raises(GatewayBindError): + validate_gateway_bind("0.0.0.0") + with pytest.raises(GatewayBindError): + validate_gateway_bind("8.8.8.8") + monkeypatch.setenv("MEMNET_GATEWAY_ALLOW_PUBLIC", "1") + validate_gateway_bind("0.0.0.0") + + +def test_stdio_unchanged_when_registry_env_is_set( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("MEMNET_GATEWAY_CONFIG", str(tmp_path / "gateway.json")) + monkeypatch.delenv("MEMNET_MCP_TRANSPORT", raising=False) + assert _transport() == "inprocess" + args = _parse_args([]) + assert args.transport == "stdio" + assert "gateway" in _parse_args(["--transport", "gateway"]).transport + resp = run_memnet(["version"]) + assert resp.exit_code == 0 + assert "0.19.18" in resp.stdout + + +def test_forward_keeps_args_and_stdin(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls: list[dict] = [] + + def fake_probe(host: str | None = None, port: int | None = None) -> bool: + del host, port + return True + + def fake_send( + args, stdin=None, host=None, port=None, timeout=None, admin_token=None, admin_usage=False + ): + calls.append( + { + "args": list(args), + "stdin": stdin, + "host": host, + "port": port, + "admin_token": admin_token, + "admin_usage": admin_usage, + } + ) + if args == ["version"]: + return {"exit_code": 0, "stdout": "@VER: memnet|0.19.18\n", "stderr": ""} + return {"exit_code": 0, "stdout": "@SESSION: mn_new|soon|60\n", "stderr": ""} + + monkeypatch.setattr("memnet_mcp.product_gateway.probe", fake_probe) + monkeypatch.setattr("memnet_mcp.product_gateway.send_command", fake_send) + cfg = tmp_path / "gw.json" + state = tmp_path / "owners.json" + _config(cfg, state, 1, 2) + gw = ProductGateway.load(str(cfg)) + gql = "MATCH (n {id: 'TSK_gw'}) SET n.status = 'settled'\n" + opened = gw.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=json.dumps( + { + "args": ["session", "open", "--map-file", "map.txt"], + "house": "syson", + "namespace": "endleaf", + } + ).encode(), + ) + assert opened.exit_code == 0 + assert calls[0]["args"] == ["version"] + assert calls[1]["args"] == ["session", "open", "--map-file", "map.txt"] + assert calls[1]["stdin"] is None + assert calls[1]["admin_token"] is None + assert calls[1]["host"] == "127.0.0.1" + assert calls[1]["port"] == 1 + mutated = gw.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=json.dumps( + { + "args": ["mutate", "--stdin", "--session", "mn_new"], + "stdin": gql, + "session": "mn_new", + } + ).encode(), + ) + assert mutated.exit_code == 0 + assert calls[-1]["args"] == ["mutate", "--stdin", "--session", "mn_new"] + assert calls[-1]["stdin"] == gql + assert "namespace" not in calls[-1] + + +def test_two_real_serves_route_isolate_and_pass_through(tmp_path: Path) -> None: + port_a = _free_port() + port_b = _free_port() + serve_a = _start_serve(port_a, tmp_path / "serve-a.log") + serve_b = _start_serve(port_b, tmp_path / "serve-b.log") + cfg = tmp_path / "gw.json" + state = tmp_path / "owners.json" + _config(cfg, state, port_a, port_b) + gw = ProductGateway.load(str(cfg)) + gw.start() + url = f"http://127.0.0.1:{gw.bound_port}/gateway" + try: + status, denied = _post(url, None, {"args": ["session", "list"]}) + assert status == 401 + assert denied["stderr"] == "@ERR: gateway_auth|bearer required\n" + status, revoked = _post(url, "endleaf-old", {"args": ["session", "list"]}) + assert status == 401 + assert "gateway_auth" in revoked["stderr"] + + before_a = _live_sids(port_a) + mismatch = ProductGateway.load(str(cfg)) + mismatch.products["endleaf"].pinned_version = "9.9.9" + bad_ver = mismatch.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=json.dumps( + {"args": ["session", "open", "--map-file", str(SCHEMA)], "namespace": "endleaf"} + ).encode(), + ) + assert bad_ver.http_status == 409 + assert bad_ver.stderr.startswith("@ERR: gateway_backend_version_mismatch|") + assert _live_sids(port_a) == before_a + + status, opened = _post( + url, + ENDLEAF, + { + "args": ["session", "open", "--map-file", str(SCHEMA)], + "house": "syson", + "namespace": "endleaf", + }, + ) + assert status == 200, opened + assert opened["exit_code"] == 0, opened + sid_a = _sid(opened) + assert sid_a in _live_sids(port_a) + assert sid_a not in _live_sids(port_b) + + snap = tmp_path / "endleaf.snap" + status, saved = _post( + url, + ENDLEAF, + { + "args": ["session", "save", "--file", str(snap), "--session", sid_a], + "session": sid_a, + }, + ) + assert status == 200 and saved["exit_code"] == 0, saved + assert snap.is_file() + status, closed_first = _post( + url, ENDLEAF, {"args": ["session", "close", sid_a], "session": sid_a} + ) + assert closed_first["exit_code"] == 0, closed_first + assert sid_a not in _live_sids(port_a) + status, loaded = _post( + url, + ENDLEAF, + { + "args": ["session", "load", "--file", str(snap), "--keep-id"], + "house": "syson", + }, + ) + assert loaded["exit_code"] == 0, loaded + assert sid_a in (loaded.get("stdout") or "") + assert sid_a in _live_sids(port_a) + + set_gql = "MATCH (n {id: 'TSK_gw'}) SET n.status = 'settled'\n" + create = ( + "CREATE (:TSK {id: 'TSK_gw', goal: 'gateway', status: 'open'})\n" + "CREATE (:TSK {id: 'TSK_gw2', goal: 'other', status: 'open'})\n" + ) + status, created = _post( + url, + ENDLEAF, + {"args": ["mutate", "--stdin", "--session", sid_a], "stdin": create, "session": sid_a}, + ) + assert created["exit_code"] == 0, created + status, settled = _post( + url, + ENDLEAF, + {"args": ["mutate", "--stdin", "--session", sid_a], "stdin": set_gql, "session": sid_a}, + ) + assert settled["exit_code"] == 0, settled + edge = ( + "MATCH (a {id: 'TSK_gw'}), (b {id: 'TSK_gw2'}) CREATE (a)-[:owns {id: 'E_gw'}]->(b)\n" + ) + status, linked = _post( + url, + ENDLEAF, + {"args": ["mutate", "--stdin", "--session", sid_a], "stdin": edge, "session": sid_a}, + ) + assert linked["exit_code"] == 0, linked + delete = "MATCH (n WHERE true)-[r {id: 'E_gw'}]->() DELETE r\n" + status, dropped = _post( + url, + ENDLEAF, + {"args": ["mutate", "--stdin", "--session", sid_a], "stdin": delete, "session": sid_a}, + ) + assert dropped["exit_code"] == 0, dropped + + missing = "MATCH (n {id: 'MISSING_GW'}) SET n.status = 'nope'\n" + miss_args = ["mutate", "--stdin", "--session", sid_a] + status, via_gw = _post( + url, ENDLEAF, {"args": miss_args, "stdin": missing, "session": sid_a} + ) + direct = send_command(miss_args, stdin=missing, host="127.0.0.1", port=port_a, timeout=10) + assert via_gw["exit_code"] == direct["exit_code"] + assert via_gw["stdout"] == (direct.get("stdout") or "") + assert via_gw["stderr"] == (direct.get("stderr") or "") + assert "@ERR:" in via_gw["stderr"] + assert "gateway_" not in via_gw["stderr"] + + wide_args = ["query", "pin-map", "--max-rows", "10000", "--session", sid_a] + status, wide = _post(url, ENDLEAF, {"args": wide_args, "session": sid_a}) + wide_direct = send_command(wide_args, host="127.0.0.1", port=port_a, timeout=10) + assert wide["exit_code"] == wide_direct["exit_code"] + assert wide["stdout"] == (wide_direct.get("stdout") or "") + assert wide["stderr"] == (wide_direct.get("stderr") or "") + + status, atelier_open = _post( + url, + ATELIER, + {"args": ["session", "open", "--map-file", str(SCHEMA)], "namespace": "atelier"}, + ) + assert atelier_open["exit_code"] == 0, atelier_open + sid_b = _sid(atelier_open) + assert sid_b in _live_sids(port_b) + assert sid_b not in _live_sids(port_a) + + status, endleaf_list = _post( + url, ENDLEAF, {"args": ["session", "list"], "namespace": "endleaf"} + ) + status, atelier_list = _post( + url, ATELIER, {"args": ["session", "list"], "namespace": "atelier"} + ) + assert sid_a in endleaf_list["stdout"] + assert sid_b not in endleaf_list["stdout"] + assert sid_b in atelier_list["stdout"] + assert sid_a not in atelier_list["stdout"] + + status, foreign = _post( + url, + ATELIER, + { + "args": ["mutate", "--stdin", "--session", sid_a], + "stdin": set_gql, + "session": sid_a, + }, + ) + assert status == 403 + assert foreign["stderr"] == "@ERR: gateway_forbidden_session|not owned by this product\n" + assert sid_a in _live_sids(port_a) + + counts_req = urllib.request.Request( + f"http://127.0.0.1:{gw.bound_port}/gateway/admin/counts", + method="GET", + headers={"Authorization": f"Bearer {ADMIN}"}, + ) + with urllib.request.urlopen(counts_req, timeout=10) as resp: + counts_body = json.loads(resp.read().decode("utf-8")) + rendered = json.dumps(counts_body) + assert "mn_" not in rendered + assert sid_a not in rendered + counts = json.loads(counts_body["stdout"]) + assert counts["products"]["endleaf"]["requests"] > 0 + assert "sessions" not in counts["products"]["endleaf"] + + _stop(serve_a) + serve_a = None + deadline = time.time() + 5 + while time.time() < deadline and probe(host="127.0.0.1", port=port_a): + time.sleep(0.05) + status, down = _post( + url, + ENDLEAF, + {"args": ["mutate", "--stdin", "--session", sid_a], "stdin": set_gql, "session": sid_a}, + ) + assert status == 502 + assert down["stderr"].startswith("@ERR: gateway_backend_unreachable|a") + assert sid_a not in _live_sids(port_b) + + status, moved = _post( + url, + ENDLEAF, + {"args": ["session", "open", "--map-file", str(SCHEMA)], "namespace": "endleaf"}, + ) + assert moved["exit_code"] == 0, moved + sid_c = _sid(moved) + assert sid_c != sid_a + assert sid_c in _live_sids(port_b) + assert sid_a not in _live_sids(port_b) + + status, closed = _post( + url, ENDLEAF, {"args": ["session", "close", sid_c], "session": sid_c} + ) + assert closed["exit_code"] == 0, closed + assert sid_c not in _live_sids(port_b) + status, gone = _post( + url, + ENDLEAF, + { + "args": [ + "session", + "save", + "--file", + str(tmp_path / "nope.snap"), + "--session", + sid_c, + ] + }, + ) + assert gone["stderr"].startswith("@ERR: gateway_forbidden_session|") + finally: + gw.stop() + _stop(serve_a) + _stop(serve_b) + + +def test_body_ceiling_is_the_only_gateway_size_gate(tmp_path: Path) -> None: + cfg = tmp_path / "gw.json" + _config(cfg, tmp_path / "owners.json", 9, 10, body_max=64) + gw = ProductGateway.load(str(cfg)) + result = gw.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=b'{"args": ["session", "list"], "stdin": "' + b"x" * 80, + ) + assert result.http_status == 413 + assert result.stderr.startswith("@ERR: gateway_body_too_large|") diff --git a/tests/test_sysml_product_gateway.py b/tests/test_sysml_product_gateway.py new file mode 100644 index 00000000..d645236e --- /dev/null +++ b/tests/test_sysml_product_gateway.py @@ -0,0 +1,68 @@ +"""Honesty-c: product gateway on memnet-mcp (MN-REQ-06.12; parent stays inventOnly).""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MODELS = ROOT / "sysml-models" / "models" +DEPLOY = MODELS / "deploy.sysml" +REQUIREMENTS = MODELS / "requirements.sysml" +VERIFY = MODELS / "verify.sysml" +IMPL = MODELS / "implementation.sysml" +NEST = ROOT / "sysml-models" / "outputs" / "product-nest-one-page.md" +CONTRACT = ROOT / "docs" / "operations" / "product-gateway-contract.md" +CODE = ROOT / "parts" / "memnet-mcp" / "software" / "memnet_mcp" / "product_gateway.py" +PROJECT = ROOT / "project.toml" + + +def test_gateway_part_and_parent_still_invent(): + text = DEPLOY.read_text(encoding="utf-8") + assert "part def MemNetProductGateway" in text + assert "part productGateway : MemNetProductGateway" in text + assert "attribute implemented : Boolean = true" in text + assert "attribute reusesTipBearer : Boolean = false" in text + assert "attribute inventOnly : Boolean = true" in text + assert "attribute implemented : Boolean = false" in text + assert "attribute codeApproved : Boolean = false" in text + assert "attribute singleBackendUnchanged : Boolean = true" in text + assert "attribute publicBindDefault : Boolean = false" in text + assert "MN_REQ_06_12_ProductGateway" in text + + +def test_requirement_verify_and_allocate(): + req = REQUIREMENTS.read_text(encoding="utf-8") + assert "MN-REQ-06.12" in req + assert "requirement def MN_REQ_06_12_ProductGateway" in req + assert "productGatewayReq" in req + ver = VERIFY.read_text(encoding="utf-8") + assert "verification def MN_VER_06_S10_ProductGateway" in ver + assert 'attribute verificationId : String = "MN-VER-06-S10"' in ver + assert "verify productGatewayReq" in ver + assert "gateway.implemented == true" in ver + assert "gateway.reusesTipBearer == false" in ver + assert "gateway.singleBackendUnchanged == true" in ver + impl = IMPL.read_text(encoding="utf-8") + assert "part def ProductGatewayMod" in impl + assert "product_gateway.py" in impl + assert "memnet_mcp.product_gateway" in impl + assert "allocation productGatewayToMod" in impl + assert CODE.is_file() + + +def test_contract_is_not_draft_and_no_semver_bump(): + contract = CONTRACT.read_text(encoding="utf-8") + assert "MN-REQ-06.12" in contract + assert "implemented for the behaviours" in contract + lowered = contract.lower() + assert "draft" not in lowered + assert "100.118.79.40" in contract + assert "18795" in contract + nest = NEST.read_text(encoding="utf-8") + assert "inventOnly" in nest + assert "MemNetLanMcpFront" in nest + assert "worthBuildingVisible=true" in nest + assert "MemNetProductGateway" in nest + project = PROJECT.read_text(encoding="utf-8") + assert 'version = "0.19.18"' in project + assert "0.19.19" not in project From 0cb5b7ac5cef355fc9025887a55ba952f44227cc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 15:31:31 +0000 Subject: [PATCH 2/3] Retrigger pull request CI. The pull request open event did not start the workflow. Co-authored-by: chouswei From 5df58daacb6564cf6b6aadf9a65c6f4bb239892d Mon Sep 17 00:00:00 2001 From: chouswei <104312007+chouswei@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:53:33 +0800 Subject: [PATCH 3/3] Gateway tests pin the running package version, not 0.19.18. Real-serve tests start serves from this tree, so pinned_version follows memnet.__version__ (0.19.19 after #204). No-bump guard follows 0.19.19. --- tests/test_product_gateway.py | 9 +++++---- tests/test_sysml_product_gateway.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_product_gateway.py b/tests/test_product_gateway.py index d3d21a9b..8a000347 100644 --- a/tests/test_product_gateway.py +++ b/tests/test_product_gateway.py @@ -15,6 +15,7 @@ import pytest +from memnet import __version__ from memnet.serve import probe, send_command from memnet_mcp.client import _transport, run_memnet from memnet_mcp.product_gateway import ( @@ -47,7 +48,7 @@ def _config( port_a: int, port_b: int, *, - pin: str = "0.19.18", + pin: str = __version__, body_max: int = 4 * 1024 * 1024, ) -> None: data = { @@ -76,7 +77,7 @@ def _config( "atelier": { "backends": ["b"], "houses": {}, - "pinned_version": "0.19.18", + "pinned_version": __version__, "credentials": [ {"id": "atelier-1", "sha256": _sha(ATELIER), "revoked": False}, ], @@ -181,7 +182,7 @@ def test_stdio_unchanged_when_registry_env_is_set( assert "gateway" in _parse_args(["--transport", "gateway"]).transport resp = run_memnet(["version"]) assert resp.exit_code == 0 - assert "0.19.18" in resp.stdout + assert __version__ in resp.stdout def test_forward_keeps_args_and_stdin(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: @@ -205,7 +206,7 @@ def fake_send( } ) if args == ["version"]: - return {"exit_code": 0, "stdout": "@VER: memnet|0.19.18\n", "stderr": ""} + return {"exit_code": 0, "stdout": f"@VER: memnet|{__version__}\n", "stderr": ""} return {"exit_code": 0, "stdout": "@SESSION: mn_new|soon|60\n", "stderr": ""} monkeypatch.setattr("memnet_mcp.product_gateway.probe", fake_probe) diff --git a/tests/test_sysml_product_gateway.py b/tests/test_sysml_product_gateway.py index d645236e..5371b2f7 100644 --- a/tests/test_sysml_product_gateway.py +++ b/tests/test_sysml_product_gateway.py @@ -64,5 +64,5 @@ def test_contract_is_not_draft_and_no_semver_bump(): assert "worthBuildingVisible=true" in nest assert "MemNetProductGateway" in nest project = PROJECT.read_text(encoding="utf-8") - assert 'version = "0.19.18"' in project - assert "0.19.19" not in project + assert 'version = "0.19.19"' in project + assert "0.19.20" not in project