From dec6064f537cb30927d24ff024a41720b52a5626 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 13:54:41 +0000 Subject: [PATCH 01/12] Add MN-REQ-01.9, 01.10, 03.4 and 05.3 for snapshot, WHERE and ACL. SysML-first honesty cut: lossless snapshot round-trip, honour-or-refuse WHERE, consistent decoded value_bytes cap, and who-check on session save/load/close. Verify cases MN-VER-01-S04, 01-S05, 03-S01, 05-S01. Co-authored-by: chouswei --- sysml-models/models/deploy.sysml | 41 +++++++++ sysml-models/models/requirements.sysml | 75 ++++++++++++++++ sysml-models/models/verify.sysml | 119 +++++++++++++++++++++++++ 3 files changed, 235 insertions(+) diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index 6caf59a..fbb9453 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -167,9 +167,16 @@ package MemNet { attribute saveOnExpireConfigurable : Boolean = true; attribute envSaveOnExpire : String = "MEMNET_SAVE_ON_EXPIRE"; attribute envExpireSnapshotDir : String = "MEMNET_EXPIRE_SNAPSHOT_DIR"; + attribute envMaxValueBytes : String = "MEMNET_MAX_VALUE_BYTES"; + attribute maxValueBytesDefault : Integer = 4096; + attribute valueBytesOnDecodedRaw : Boolean = true; + attribute gqlMutateEnforcesValueBytes : Boolean = true; + attribute sessionSaveLoadCloseAclWho : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_1_StoreResourceCaps; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_2_QueryFanoutCaps; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; satisfy MN_REQ_00_MissionBridge::MN_REQ_10_LlmPropertiesAndLimits::MN_REQ_10_2_PinMapMustFitContext; satisfy MN_REQ_00_MissionBridge::MN_REQ_12_MultitaskMode::MN_REQ_12_7_NoAssumeAclReserveIngest; } @@ -424,10 +431,16 @@ package MemNet { attribute libraryIngestThisCut : Boolean = true; part allocator : IdAllocator; part caps : CapsPolicy; + attribute honourWhereOrRefuse : Boolean = true; + attribute whereTrueEdgeDeleteKept : Boolean = true; + attribute matchMapEqualityKept : Boolean = true; + attribute gqlValueBytesHardCap : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_1_AddFailsIfExists; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_2_UpdateFailsIfAbsent; satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_3_NoSilentUpsert; + satisfy MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_4_HonourWherePredicate; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; satisfy MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph::MN_REQ_02_7_SchemaValidatedIngest; satisfy MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph::MN_REQ_02_8_IdNotRequiredOnWire; satisfy MN_REQ_00_MissionBridge::MN_REQ_12_MultitaskMode::MN_REQ_12_7_NoAssumeAclReserveIngest; @@ -1072,11 +1085,18 @@ package MemNet { attribute notWholeCabinetDump : Boolean = true; attribute loadByKnownSid : Boolean = true; attribute expireMissHonest : Boolean = true; + attribute losslessPropertyRoundTrip : Boolean = true; + attribute failClosedUnsaveable : Boolean = true; + attribute expireSaveUnsaveableNoFile : Boolean = true; + attribute legacy01918Load : Boolean = true; + attribute escapeLfCrPipeBackslash : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_6_OptionalKeepSessionId; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_3_SessionTtlAndCaps; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; satisfy MN_REQ_00_MissionBridge::MN_REQ_11_SnapshotInterop::MN_REQ_11_4_DistinctFromSessionSnapshot; } @@ -1357,8 +1377,11 @@ package MemNet { attribute implemented : Boolean = true; attribute productCommand : Boolean = true; attribute dumpsS : Boolean = false; + attribute acceptsCaller : Boolean = true; + attribute aclWhoWhenEnabled : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_2_SessionLifecycleOps; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_5_NoStoreKeyToolSurface; } @@ -1373,6 +1396,13 @@ package MemNet { attribute saveOnExpireConfigurable : Boolean = true; attribute saveOnExpireDefault : Boolean = false; attribute ramDropsAfterExpireSave : Boolean = true; + attribute acceptsCaller : Boolean = true; + attribute aclWhoWhenEnabled : Boolean = true; + attribute failClosedUnsaveable : Boolean = true; + + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; } part def CmdSessionLoad { @@ -1392,6 +1422,13 @@ package MemNet { attribute loadByKnownSid : Boolean = true; attribute expireSnapKeepId : Boolean = true; attribute mustNotEchoSid : Boolean = true; + attribute acceptsCaller : Boolean = true; + attribute aclWhoWhenEnabled : Boolean = true; + attribute legacy01918Load : Boolean = true; + + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; } part def CmdFind { @@ -1618,6 +1655,8 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_6_OptionalKeepSessionId; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_1_GenericToolSurface; satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_5_NoStoreKeyToolSurface; satisfy MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph::MN_REQ_02_8_IdNotRequiredOnWire; @@ -1709,6 +1748,8 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_6_OptionalKeepSessionId; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_1_InProcessPrimary; satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_1_GenericToolSurface; satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_2_StructuredToolEnvelope; diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index cc972b8..fec7c44 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -26,6 +26,10 @@ MN-REQ-06.8 Tip MemNet access portal (ops): admin invite, Google login, Bearer for keyed tip MCP; tip≠face MN-REQ-06.9 LAN MCP front invent (#191): ClusterRoute — where the session lives; one MCP catalogue, N LAN serves; cousin of #47; inventOnly MN-REQ-06.10 SliceHandCarry invent (#47 cousin): bounded WorkingMemorySlice copy into another session; not a live hop; inventOnly + MN-REQ-01.9 Snapshot save/load lossless property round-trip; fail-closed save + MN-REQ-01.10 CapsPolicy who-check on session save / load-into-ACL / close + MN-REQ-03.4 MATCH WHERE SET/DELETE SHALL honour the predicate or refuse + MN-REQ-05.3 One decoded value_bytes cap on pipe mutate, GQL mutate, snapshot load MN-REQ-06.11 Admin-only serve usage report (counts/caps; opaque session alias; not agent MCP) MN-REQ-11.17 Catalog Snap / model Snap — session strata (0.15) MN-REQ-11.17.1 Cross-cut satisfy locators on the catalog; SysMLEdge @@ -204,6 +208,34 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-01.8"; } + + requirement def MN_REQ_01_9_SnapshotLosslessRoundTrip { + doc /* + Anything the write path accepts SHALL session_save and + session_load byte-for-byte as the same property values. + That includes LF, CR, backslash, both quote kinds, dollar, + braces, pipe, and CJK. Snapshot emit SHALL escape those + losslessly; load SHALL decode them. Save SHALL fail closed + (snapshot_unsaveable naming the row) rather than write an + unloadable file. Expire-save that cannot round-trip SHALL + NOT write a file; RAM still drops; next use is + session_expired|snap_missing. Snapshots written by 0.19.18 + (pipe and backslash escapes only) SHALL still load. + */ + attribute requirementId : String = "MN-REQ-01.9"; + } + + requirement def MN_REQ_01_10_SessionLifecycleAclWho { + doc /* + When session ACL is enabled, session_save, session_load into + that ACL'd session, and session_close SHALL accept --caller / + MEMNET_CALLER and enforce CapsPolicy who-check with the same + codes as pin_map / mutate (acl_who, acl_denied, acl_forbidden). + MCP SHALL pass caller through on those tools. Without ACL, + behaviour is unchanged. + */ + attribute requirementId : String = "MN-REQ-01.10"; + } } // ----- Memory net information model: GQL node, edge, property ----- @@ -381,6 +413,21 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-03.3"; } + + requirement def MN_REQ_03_4_HonourWherePredicate { + doc /* + MATCH ... WHERE ... SET and MATCH ... WHERE ... DELETE SHALL + NOT ignore the WHERE predicate. If lowering cannot honour it, + the whole statement SHALL refuse with unsupported_predicate + naming that predicate, and SHALL apply nothing. Forms that + work today SHALL remain: MATCH (n WHERE true)-[r {id}]->() + DELETE r (edge delete); property-map equality in MATCH. + Honoured WHERE forms (equality, CONTAINS, STARTS WITH, + ENDS WITH, =~, IN, true/false, AND/OR/NOT of those) SHALL + actually filter. + */ + attribute requirementId : String = "MN-REQ-03.4"; + } } // ----- Slice economy (pin map / recycle / walk) ----- @@ -604,6 +651,22 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-05.2"; } + + requirement def MN_REQ_05_3_ConsistentValueByteCap { + doc /* + The field-value byte cap SHALL be one hard cap on leftover + pipe mutate, GQL mutate, and snapshot load, measured as the + UTF-8 byte length of the decoded (raw) property value. + Default 4096. Knob MEMNET_MAX_VALUE_BYTES (a product MAY + raise it to 16384). Over-cap SHALL refuse + limit_exceeded|value_bytes n/max and SHALL NOT accept the + write. An escaped snapshot value whose decoded size is under + this cap SHALL NOT trip line_bytes on load. line_bytes + remains a leftover-pipe / snapshot decoded-line cap + (MEMNET_MAX_LINE_BYTES, default 32768). + */ + attribute requirementId : String = "MN-REQ-05.3"; + } } // ----- Process / transport boundary ----- @@ -1868,6 +1931,10 @@ package MemNetRequirements { : MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_7_SessionAsSsotHandle; requirement noGraphDumpHandoffReq : MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_8_NoGraphDumpHandoff; + requirement snapshotLosslessRoundTripReq + : MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_9_SnapshotLosslessRoundTrip; + requirement sessionLifecycleAclWhoReq + : MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_10_SessionLifecycleAclWho; requirement memoryNetGraphReq : MN_REQ_00_MissionBridge::MN_REQ_02_MemoryNetGraph; @@ -1898,6 +1965,8 @@ package MemNetRequirements { : MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_2_UpdateFailsIfAbsent; requirement noSilentUpsertReq : MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_3_NoSilentUpsert; + requirement honourWherePredicateReq + : MN_REQ_00_MissionBridge::MN_REQ_03_StrictMutate::MN_REQ_03_4_HonourWherePredicate; requirement sliceEconomyReq : MN_REQ_00_MissionBridge::MN_REQ_04_SliceEconomy; @@ -1930,6 +1999,8 @@ package MemNetRequirements { : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_1_StoreResourceCaps; requirement queryFanoutCapsReq : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_2_QueryFanoutCaps; + requirement consistentValueByteCapReq + : MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_3_ConsistentValueByteCap; requirement processBoundaryReq : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary; @@ -2132,6 +2203,8 @@ package MemNetRequirements { end #derive ::> optionalKeepSessionIdReq; end #derive ::> sessionAsSsotHandleReq; end #derive ::> noGraphDumpHandoffReq; + end #derive ::> snapshotLosslessRoundTripReq; + end #derive ::> sessionLifecycleAclWhoReq; } #derivation connection deriveMemoryNetGraphLeaves { @@ -2152,6 +2225,7 @@ package MemNetRequirements { end #derive ::> addFailsIfExistsReq; end #derive ::> updateFailsIfAbsentReq; end #derive ::> noSilentUpsertReq; + end #derive ::> honourWherePredicateReq; } #derivation connection deriveSliceEconomyLeaves { @@ -2173,6 +2247,7 @@ package MemNetRequirements { end #original ::> hardCapsReq; end #derive ::> storeResourceCapsReq; end #derive ::> queryFanoutCapsReq; + end #derive ::> consistentValueByteCapReq; } #derivation connection deriveProcessBoundaryLeaves { diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 08a67b9..f10a828 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -58,6 +58,11 @@ grain not per-leaf; budget pre-check; caller-gone rollback; replace. Expire session_save (MN-VER-01-S03): configurable; default off; RAM drops; file until user drop; load restores; not Neo4j. + Snapshot lossless round-trip (MN-VER-01-S04): emit escapes LF/CR; + fail-closed save; 0.19.18 snapshots still load. + Session save/load/close ACL who (MN-VER-01-S05) when session ACL on. + Honour WHERE or refuse (MN-VER-03-S01). + Consistent decoded value_bytes cap (MN-VER-05-S01). */ package MemNetVerification { private import ScalarValues::*; @@ -1853,6 +1858,112 @@ package MemNetVerification { } } + verification def MN_VER_01_S04_SnapshotLosslessRoundTrip { + doc /* + MN-REQ-01.9 / MN-REQ-05.3 — snapshot emit escapes LF/CR/pipe/backslash + losslessly; save fails closed; expire-save writes no unloadable file; + 0.19.18 snapshots still load; value_bytes is decoded raw UTF-8. + */ + attribute verificationId : String = "MN-VER-01-S04"; + + subject system : MemNetSystem; + + objective snapshotLosslessRoundTrip { + verify snapshotLosslessRoundTripReq; + verify consistentValueByteCapReq; + require constraint { + system.core.transport.inProcess.memory.sessions.snap + .losslessPropertyRoundTrip == true + and system.core.transport.inProcess.memory.sessions.snap + .failClosedUnsaveable == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveUnsaveableNoFile == true + and system.core.transport.inProcess.memory.sessions.snap + .legacy01918Load == true + and system.core.transport.inProcess.memory.sessions.snap + .escapeLfCrPipeBackslash == true + and system.core.cli.sessionSave.failClosedUnsaveable == true + and system.core.cli.sessionLoad.legacy01918Load == true + and system.core.transport.inProcess.memory.sessions.caps + .valueBytesOnDecodedRaw == true + and system.core.transport.inProcess.memory.sessions.caps + .gqlMutateEnforcesValueBytes == true + } + } + } + + verification def MN_VER_01_S05_SessionLifecycleAclWho { + doc /* + MN-REQ-01.10 — session save / load-into-ACL / close who-check when + session ACL is enabled. --caller / MEMNET_CALLER. Same codes. + */ + attribute verificationId : String = "MN-VER-01-S05"; + + subject system : MemNetSystem; + + objective sessionLifecycleAclWho { + verify sessionLifecycleAclWhoReq; + require constraint { + system.core.cli.sessionSave.acceptsCaller == true + and system.core.cli.sessionSave.aclWhoWhenEnabled == true + and system.core.cli.sessionLoad.acceptsCaller == true + and system.core.cli.sessionLoad.aclWhoWhenEnabled == true + and system.core.cli.sessionClose.acceptsCaller == true + and system.core.cli.sessionClose.aclWhoWhenEnabled == true + and system.core.transport.inProcess.memory.sessions.caps + .sessionSaveLoadCloseAclWho == true + } + } + } + + verification def MN_VER_03_S01_HonourWherePredicate { + doc /* + MN-REQ-03.4 — MATCH WHERE SET/DELETE honours the predicate or + refuses unsupported_predicate with nothing applied. + */ + attribute verificationId : String = "MN-VER-03-S01"; + + subject system : MemNetSystem; + + objective honourWherePredicate { + verify honourWherePredicateReq; + require constraint { + system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.honourWhereOrRefuse == true + and system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.whereTrueEdgeDeleteKept == true + and system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.matchMapEqualityKept == true + } + } + } + + verification def MN_VER_05_S01_ConsistentValueByteCap { + doc /* + MN-REQ-05.3 — one decoded value_bytes cap; knob MEMNET_MAX_VALUE_BYTES; + default 4096; GQL mutate refuses over-cap. + */ + attribute verificationId : String = "MN-VER-05-S01"; + + subject system : MemNetSystem; + + objective consistentValueByteCap { + verify consistentValueByteCapReq; + require constraint { + system.core.transport.inProcess.memory.sessions.caps + .envMaxValueBytes == "MEMNET_MAX_VALUE_BYTES" + and system.core.transport.inProcess.memory.sessions.caps + .maxValueBytesDefault == 4096 + and system.core.transport.inProcess.memory.sessions.caps + .valueBytesOnDecodedRaw == true + and system.core.transport.inProcess.memory.sessions.caps + .gqlMutateEnforcesValueBytes == true + and system.core.transport.inProcess.memory.sessions.recallCommit + .commit.mutate.gqlValueBytesHardCap == true + } + } + } + verification def MN_VER_06_S06_StorageRoles { doc /* storageRole only where bytes outlive the call: working_memory, @@ -1942,6 +2053,14 @@ package MemNetVerification { : MN_VER_06_S04_SsotToCodeAllocate; verification expireSaveConfigurableVerify : MN_VER_01_S03_ExpireSaveConfigurable; + verification snapshotLosslessRoundTripVerify + : MN_VER_01_S04_SnapshotLosslessRoundTrip; + verification sessionLifecycleAclWhoVerify + : MN_VER_01_S05_SessionLifecycleAclWho; + verification honourWherePredicateVerify + : MN_VER_03_S01_HonourWherePredicate; + verification consistentValueByteCapVerify + : MN_VER_05_S01_ConsistentValueByteCap; verification storageRolesVerify : MN_VER_06_S06_StorageRoles; verification lanMcpFrontSeveralServesVerify : MN_VER_06_S07_LanMcpFrontSeveralServes; From 82b116a13ece18ef5fdd21e1d884f817094020f4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 13:54:46 +0000 Subject: [PATCH 02/12] Fix snapshot round-trip, GQL WHERE honour-or-refuse, and ACL who-check. Snapshot emit escapes LF/CR/pipe/backslash; load decodes; save fails closed with snapshot_unsaveable. GQL mutate shares MEMNET_MAX_VALUE_BYTES on decoded UTF-8. MATCH WHERE SET/DELETE honours the predicate or refuses unsupported_predicate with nothing applied. Session save/load/close accept --caller when ACL is enabled; MCP passes caller through. Co-authored-by: chouswei --- parts/common/memnet/memnet/cli.py | 48 +++- parts/common/memnet/memnet/gql.py | 252 +++++++++++++++++- parts/common/memnet/memnet/mutate_gate.py | 43 ++- parts/common/memnet/memnet/session.py | 3 + parts/common/memnet/memnet/snapshot.py | 104 ++++++-- parts/common/memnet/memnet/tag_map.py | 41 ++- parts/common/memnet/memnet/tier_a.py | 1 + parts/common/memnet/memnet/wire.py | 15 +- .../memnet-mcp/software/memnet_mcp/server.py | 16 +- 9 files changed, 475 insertions(+), 48 deletions(-) diff --git a/parts/common/memnet/memnet/cli.py b/parts/common/memnet/memnet/cli.py index efd058c..5be5c5d 100644 --- a/parts/common/memnet/memnet/cli.py +++ b/parts/common/memnet/memnet/cli.py @@ -128,6 +128,20 @@ def _handle_error(exc: MemNetError) -> None: raise typer.Exit(exc.exit_code) from exc +def _acl_check(ss, caller: str | None, permission: str) -> None: + from memnet.acl import check_permission + + try: + check_permission( + ss.acl, + caller=caller or os.environ.get("MEMNET_CALLER"), + permission=permission, # type: ignore[arg-type] + agent=os.environ.get("MEMNET_AGENT"), + ) + except MemNetError as exc: + _handle_error(exc) + + def _load_session(session: str | None, *, exclusive: bool = False): purge_expired(_caps()) try: @@ -396,6 +410,10 @@ def session_list() -> None: def session_save( file: Annotated[Path, typer.Option("--file", help="User snapshot path (wire format)")], session: Annotated[str | None, typer.Option("--session")] = None, + caller: Annotated[ + str | None, + typer.Option("--caller", help="CallerId for CapsPolicy ACL who-check"), + ] = None, ) -> None: """Write the session graph. After TTL, only if MEMNET_SAVE_ON_EXPIRE.""" try: @@ -404,9 +422,14 @@ def session_save( except MemNetError as exc: _handle_error(exc) raise AssertionError("unreachable") from exc + _acl_check(ss, caller, "pin_map") reset_warn_budget() with ss.lock(exclusive=True): - rows = write_snapshot(ss, file) + try: + rows = write_snapshot(ss, file) + except MemNetError as exc: + _handle_error(exc) + raise AssertionError("unreachable") from exc if expired: remove_entry(sid) emit_wrn("session_expired_saved", str(file)) @@ -426,9 +449,22 @@ def session_load( typer.Option("--keep-id", help="Reuse session id from snapshot"), ] = False, session: Annotated[str | None, typer.Option("--session")] = None, + caller: Annotated[ + str | None, + typer.Option("--caller", help="CallerId for CapsPolicy ACL who-check"), + ] = None, ) -> None: """Load a snapshot. ``--file`` or expire-dir load by ``--session`` (known sid).""" caps = _caps() + if file is None: + sid_hint = session or os.environ.get("MEMNET_SESSION") + if sid_hint: + entry = get_entry(sid_hint) + acl = getattr(entry, "acl", None) if entry is not None else None + if acl is not None and acl.enabled: + from types import SimpleNamespace + + _acl_check(SimpleNamespace(acl=acl), caller, "pin_map") purge_expired(caps) try: if file is not None: @@ -496,8 +532,16 @@ def session_expire_status() -> None: @session_app.command("close") -def session_close(session_id: str) -> None: +def session_close( + session_id: str, + caller: Annotated[ + str | None, + typer.Option("--caller", help="CallerId for CapsPolicy ACL who-check"), + ] = None, +) -> None: try: + ss = get_session(session_id, _caps()) + _acl_check(ss, caller, "mutate") close_session(session_id, _caps()) emit_session(session_id, "closed") except MemNetError as exc: diff --git a/parts/common/memnet/memnet/gql.py b/parts/common/memnet/memnet/gql.py index d870c44..d769f07 100644 --- a/parts/common/memnet/memnet/gql.py +++ b/parts/common/memnet/memnet/gql.py @@ -97,6 +97,219 @@ class _NodePattern: var: str | None = None label: str | None = None props: dict[str, Any] = field(default_factory=dict) + where: WherePred | None = None + + +@dataclass +class WherePred: + """Honoured MATCH WHERE tree. Unknown ops refuse at parse (MN-REQ-03.4).""" + + op: str + key: str = "" + value: str = "" + var: str = "" + children: list[WherePred] = field(default_factory=list) + + +def _unsupported_pred(name: str, line_no: int | None = None) -> None: + raise ParseError(f"WHERE {name} is not honoured", line_no, code="unsupported_predicate") + + +def parse_where_clause(text: str, line_no: int | None = None) -> WherePred: + parser = _WhereParser(text, line_no) + pred = parser.parse() + parser.skip_ws() + if parser.i < len(parser.s): + _unsupported_pred(parser.s[parser.i : parser.i + 24].strip() or "clause", line_no) + return pred + + +class _WhereParser: + def __init__(self, text: str, line_no: int | None) -> None: + self.s = text + self.i = 0 + self.line_no = line_no + + def skip_ws(self) -> None: + self.i = _skip_ws(self.s, self.i) + + def peek_kw(self, word: str) -> bool: + self.skip_ws() + n = len(word) + if self.s[self.i : self.i + n].upper() != word.upper(): + return False + end = self.i + n + if end < len(self.s) and (self.s[end].isalnum() or self.s[end] == "_"): + return False + return True + + def take_kw(self, word: str) -> bool: + if not self.peek_kw(word): + return False + self.i += len(word) + return True + + def parse(self) -> WherePred: + return self._or() + + def _or(self) -> WherePred: + left = self._and() + while self.take_kw("OR"): + right = self._and() + left = WherePred(op="OR", children=[left, right]) + return left + + def _and(self) -> WherePred: + left = self._not() + while self.take_kw("AND"): + right = self._not() + left = WherePred(op="AND", children=[left, right]) + return left + + def _not(self) -> WherePred: + if self.take_kw("NOT"): + return WherePred(op="NOT", children=[self._not()]) + if self.take_kw("XOR"): + _unsupported_pred("XOR", self.line_no) + return self._primary() + + def _primary(self) -> WherePred: + self.skip_ws() + if self.i < len(self.s) and self.s[self.i] == "(": + self.i += 1 + inner = self.parse() + self.skip_ws() + if self.i >= len(self.s) or self.s[self.i] != ")": + _unsupported_pred("parenthesis", self.line_no) + self.i += 1 + return inner + if self.take_kw("TRUE"): + return WherePred(op="TRUE") + if self.take_kw("FALSE"): + return WherePred(op="FALSE") + return self._comparison() + + def _prop_ref(self) -> tuple[str, str] | None: + self.skip_ws() + m = re.match(rf"({_IDENT})\.({_IDENT})", self.s[self.i :]) + if not m: + return None + self.i += m.end() + return m.group(1), m.group(2) + + def _comparison(self) -> WherePred: + self.skip_ws() + start = self.i + # value IN n.key + try: + val, j = _parse_value(self.s, self.i) + saved = self.i + self.i = j + if self.take_kw("IN"): + pref = self._prop_ref() + if pref is None: + _unsupported_pred("IN", self.line_no) + assert pref is not None + var, key = pref + return WherePred(op="IN", var=var, key=key, value=_value_to_store(val)) + self.i = saved + except ParseError: + self.i = start + pref = self._prop_ref() + if pref is None: + snippet = self.s[self.i : self.i + 24].strip() or "predicate" + _unsupported_pred(snippet, self.line_no) + raise AssertionError("unreachable") + var, key = pref + self.skip_ws() + if self.take_kw("STARTS"): + if not self.take_kw("WITH"): + _unsupported_pred("STARTS", self.line_no) + op = "STARTS" + elif self.take_kw("ENDS"): + if not self.take_kw("WITH"): + _unsupported_pred("ENDS", self.line_no) + op = "ENDS" + elif self.take_kw("CONTAINS"): + op = "CONTAINS" + elif self.s[self.i : self.i + 2] == "=~": + self.i += 2 + op = "REGEX" + elif self.s[self.i : self.i + 2] == "<>": + self.i += 2 + op = "NE" + elif self.s[self.i : self.i + 2] == "!=": + self.i += 2 + op = "NE" + elif self.i < len(self.s) and self.s[self.i] == "=": + self.i += 1 + op = "EQ" + elif self.i < len(self.s) and self.s[self.i] in "<>": + _unsupported_pred(self.s[self.i], self.line_no) + raise AssertionError("unreachable") + else: + snippet = self.s[self.i : self.i + 16].strip() or "predicate" + _unsupported_pred(snippet, self.line_no) + raise AssertionError("unreachable") + val, self.i = _parse_value(self.s, self.i) + if op == "REGEX": + pattern = _value_to_store(val) + try: + re.compile(pattern) + except re.error: + _unsupported_pred("=~", self.line_no) + return WherePred(op=op, var=var, key=key, value=_value_to_store(val)) + + +def eval_where(rec: Any, pred: WherePred) -> bool: + """Whether *rec* satisfies a honoured WHERE tree.""" + op = pred.op + if op == "TRUE": + return True + if op == "FALSE": + return False + if op == "AND": + return all(eval_where(rec, c) for c in pred.children) + if op == "OR": + return any(eval_where(rec, c) for c in pred.children) + if op == "NOT": + return not eval_where(rec, pred.children[0]) + raw = str(rec.fields.get(pred.key, "")) + if op == "EQ": + return raw == pred.value + if op == "NE": + return raw != pred.value + if op == "CONTAINS": + return pred.value in raw + if op == "STARTS": + return raw.startswith(pred.value) + if op == "ENDS": + return raw.endswith(pred.value) + if op == "REGEX": + return re.search(pred.value, raw) is not None + if op == "IN": + return pred.value in _field_as_list(raw) + return False + + +def _field_as_list(raw: str) -> list[str]: + text = raw.strip() + if text.startswith("["): + try: + val = json.loads(text) + if isinstance(val, list): + return [str(x) for x in val] + except json.JSONDecodeError: + pass + if not text: + return [] + return [p.strip() for p in text.split(",") if p.strip()] + + +def _where_true_only(stmt: str) -> bool: + """True when every WHERE in *stmt* is the tautology ``true``.""" + stripped = re.sub(r"\bWHERE\s+true\b", " ", stmt, flags=re.IGNORECASE) + return re.search(r"\bWHERE\b", stripped, re.IGNORECASE) is None def looks_like_gql(line: str) -> bool: @@ -405,10 +618,19 @@ def _parse_node_patterns(chunk: str) -> tuple[list[_NodePattern], int, str]: if i < len(s) and s[i] == "{": props, i = _parse_map(s, i) i = _skip_ws(s, i) + inline_where: WherePred | None = None + if re.match(r"WHERE\b", s[i:], re.IGNORECASE): + mw = re.match(r"WHERE\b", s[i:], re.IGNORECASE) + assert mw is not None + wp = _WhereParser(s, None) + wp.i = i + mw.end() + inline_where = wp.parse() + i = wp.i + i = _skip_ws(s, i) if i >= len(s) or s[i] != ")": raise ParseError("expected ')' after node pattern") i += 1 - patterns.append(_NodePattern(var=var, label=label, props=props)) + patterns.append(_NodePattern(var=var, label=label, props=props, where=inline_where)) i = _skip_ws(s, i) if i < len(s) and s[i] == ",": i += 1 @@ -638,10 +860,28 @@ def _parse_match(s: str, line_no: int) -> list[NodeRec | EdgeRec]: raise ParseError("bad MATCH", line_no) patterns_chunk, rest = _split_match_body(m.group(1)) try: - patterns, _consumed, _full = _parse_node_patterns(patterns_chunk) + patterns, consumed, full = _parse_node_patterns(patterns_chunk) except ParseError: # Relationship MATCH for delete: ()-[r {id:…}]-() return _parse_match_rel_delete(s, line_no, rest) + tail = full[consumed:].strip() + if tail.startswith("-") or tail.startswith("<-"): + return _parse_match_rel_delete(s, line_no, rest) + where_pred: WherePred | None = None + if tail.upper().startswith("WHERE"): + where_pred = parse_where_clause(tail[5:].strip(), line_no) + elif tail: + raise ParseError(f"unsupported MATCH continuation: {tail[:60]!r}", line_no) + inline_preds = [p.where for p in patterns if p.where is not None] + if inline_preds: + combined = inline_preds[0] + for extra in inline_preds[1:]: + combined = WherePred(op="AND", children=[combined, extra]) + where_pred = ( + WherePred(op="AND", children=[combined, where_pred]) + if where_pred is not None + else combined + ) if not patterns and not rest: raise ParseError("MATCH needs node patterns", line_no) @@ -657,6 +897,8 @@ def _parse_match(s: str, line_no: int) -> list[NodeRec | EdgeRec]: rest_u = rest.upper() if rest_u.startswith("CREATE"): + if where_pred is not None and where_pred.op != "TRUE": + _unsupported_pred("predicate on MATCH CREATE", line_no) cm = _RE_CREATE_REL.match(rest) if not cm: raise ParseError( @@ -689,6 +931,8 @@ def _parse_match(s: str, line_no: int) -> list[NodeRec | EdgeRec]: if rest_u.startswith("SET"): absorb = _parse_same_thing_set(s, rest, patterns, var_pat, line_no) if absorb is not None: + if where_pred is not None and where_pred.op != "TRUE": + _unsupported_pred("predicate on SameThingAbsorb", line_no) return [absorb] if len(patterns) != 1: raise ParseError( @@ -707,6 +951,7 @@ def _parse_match(s: str, line_no: int) -> list[NodeRec | EdgeRec]: fields=fields, raw=s, match_props=_props_as_str(p.props), + where=where_pred, ) ] @@ -723,6 +968,7 @@ def _parse_match(s: str, line_no: int) -> list[NodeRec | EdgeRec]: fields=[], raw=s, match_props=_props_as_str(p.props), + where=where_pred, ) ] @@ -780,6 +1026,8 @@ def _parse_same_thing_set( def _parse_match_rel_delete(s: str, line_no: int, rest: str) -> list[NodeRec | EdgeRec]: """MATCH ()-[r {id:'E1'}]-() DELETE r (simplified gated form).""" + if not _where_true_only(s): + _unsupported_pred("predicate on relationship DELETE", line_no) m = re.search( rf"\[\s*(?:({_IDENT})\s*)?(?::({_RELTYPE}))?\s*(\{{[^{{}}]*\}})?\s*\]", s, diff --git a/parts/common/memnet/memnet/mutate_gate.py b/parts/common/memnet/memnet/mutate_gate.py index 9ff7904..b0e099b 100644 --- a/parts/common/memnet/memnet/mutate_gate.py +++ b/parts/common/memnet/memnet/mutate_gate.py @@ -18,6 +18,7 @@ from memnet.models import Record from memnet.output import emit_record from memnet.same_thing_absorb import absorb_same_thing +from memnet.tag_map import check_value_bytes from memnet.tier_a import EdgeRec, Field, NodeRec, Op, Section _MERGE_TRUE = frozenset({"true", "1", "yes"}) @@ -689,19 +690,27 @@ def _item_to_record(self, it: NodeRec | EdgeRec) -> Record: def _pattern_hits(self, it: NodeRec) -> list[Record]: store = self.ss.store + hits: list[Record] | None = None if it.id and it.id in store._by_hid: - return [store._by_hid[it.id]] - props = dict(it.match_props or {}) - if it.id and "id" not in props: - one = store.resolve_one(it.id) - if one is not None and not props: - return [one] - if it.id: - props["id"] = it.id - tag = it.kind or None - if not tag and not props: - return [] - return store.match_nodes(tag=tag, props=props) + hits = [store._by_hid[it.id]] + else: + props = dict(it.match_props or {}) + if it.id and "id" not in props: + one = store.resolve_one(it.id) + if one is not None and not props: + hits = [one] + elif it.id: + props["id"] = it.id + if hits is None: + tag = it.kind or None + if not tag and not props and it.where is None: + return [] + hits = store.match_nodes(tag=tag, props=props) + if it.where is not None: + from memnet.gql import eval_where + + hits = [h for h in hits if eval_where(h, it.where)] + return hits def _same_thing_pair(self, it: NodeRec) -> tuple[Record, Record]: keep_hits = self._pattern_hits( @@ -876,6 +885,11 @@ def _node_to_record(self, node: NodeRec) -> Record: fields.setdefault(fname, base.get(fname, "")) if not fields.get("id"): fields.pop("id", None) + caps = getattr(self.ss, "caps", None) + if caps is not None: + for val in fields.values(): + if val: + check_value_bytes(val, caps) rec = Record(tag=kind, fields=fields) if bound is not None: rec.hid = bound.hid @@ -921,6 +935,11 @@ def _edge_to_record(self, edge: EdgeRec) -> Record: fields.setdefault(fname, "") if not fields.get("id"): fields.pop("id", None) + caps = getattr(self.ss, "caps", None) + if caps is not None: + for val in fields.values(): + if val: + check_value_bytes(val, caps) rec = Record(tag="EDG", fields=fields) if existing is not None: rec.hid = existing.hid diff --git a/parts/common/memnet/memnet/session.py b/parts/common/memnet/memnet/session.py index 29169da..0caf62b 100644 --- a/parts/common/memnet/memnet/session.py +++ b/parts/common/memnet/memnet/session.py @@ -247,6 +247,9 @@ def snapshot_expired_session(session_id: str, caps: Caps | None = None) -> str | except OSError as exc: emit_wrn("expire_snapshot_failed", type(exc).__name__) return None + except MemNetError as exc: + emit_wrn("expire_snapshot_failed", exc.code) + return None emit_wrn("expire_snapshot", "written") return str(path) diff --git a/parts/common/memnet/memnet/snapshot.py b/parts/common/memnet/memnet/snapshot.py index a99c743..c8a756b 100644 --- a/parts/common/memnet/memnet/snapshot.py +++ b/parts/common/memnet/memnet/snapshot.py @@ -43,26 +43,18 @@ def _snapshot_emit_nick(rec: Record, used: set[str]) -> str: return leftover_wire_nick(rec.hid, kind=rec.tag, used=used) -def snapshot_text(ss: SessionStore) -> str: - lines = [SNAPSHOT_MAGIC] - m = ss.meta - hw = "1" if m.has_writes else "0" - modified = m.modified_at or "-" - lines.append( - f"@SNAP: 1|{m.session_id}|{m.created_at}|{m.expires_at}|{m.ttl_minutes}|{hw}|{modified}" - ) - lines.append(_SECTION_MAP) - lines.extend(tag_map_to_lines(ss.tag_map)) - lines.append(_SECTION_REL) - for rel in sorted(ss.relations): - lines.append(f"@REL: {rel}") - lines.append(_SECTION_REC) +def _nick_of(rec: Record) -> str: + return rec.fields.get("id") or rec.tag + + +def _emit_record_lines(ss: SessionStore) -> tuple[list[str], dict[str, str]]: used: set[str] = set() hid_to_nick: dict[str, str] = {} for rid in ss.store.write_order: rec = ss.store._by_hid.get(rid) if rec: hid_to_nick[rec.hid] = _snapshot_emit_nick(rec, used) + rec_lines: list[str] = [] for rid in ss.store.write_order: rec = ss.store._by_hid.get(rid) if not rec: @@ -75,10 +67,88 @@ def snapshot_text(ss: SessionStore) -> str: if token in hid_to_nick: fields[key] = hid_to_nick[token] clone = rec.model_copy(update={"fields": fields}) - lines.append(emit_record(clone, ss.tag_map)) + rec_lines.append(emit_record(clone, ss.tag_map)) + return rec_lines, hid_to_nick + + +def snapshot_text(ss: SessionStore) -> str: + rec_lines, hid_to_nick = _emit_record_lines(ss) + _verify_emitted_rows(ss, rec_lines, hid_to_nick) + return _format_snapshot(ss, rec_lines) + + +def _format_snapshot(ss: SessionStore, rec_lines: list[str]) -> str: + lines = [SNAPSHOT_MAGIC] + m = ss.meta + hw = "1" if m.has_writes else "0" + modified = m.modified_at or "-" + lines.append( + f"@SNAP: 1|{m.session_id}|{m.created_at}|{m.expires_at}|{m.ttl_minutes}|{hw}|{modified}" + ) + lines.append(_SECTION_MAP) + lines.extend(tag_map_to_lines(ss.tag_map)) + lines.append(_SECTION_REL) + for rel in sorted(ss.relations): + lines.append(f"@REL: {rel}") + lines.append(_SECTION_REC) + lines.extend(rec_lines) return "\n".join(lines) + "\n" +def _verify_emitted_rows( + ss: SessionStore, + rec_lines: list[str], + hid_to_nick: dict[str, str], +) -> None: + """Refuse save if any emitted row would not load as the same values.""" + used_nicks: set[str] = set() + rec_iter = iter(rec_lines) + for rid in ss.store.write_order: + rec = ss.store._by_hid.get(rid) + if not rec: + continue + try: + line = next(rec_iter) + except StopIteration as exc: + raise MemNetError( + "snapshot_unsaveable", + f"{rec.tag} nick={_nick_of(rec)} missing emit row", + ) from exc + try: + parsed = parse_line(line, ss.tag_map, ss.caps, used_nicks=used_nicks) + except MemNetError as exc: + raise MemNetError( + "snapshot_unsaveable", + f"{rec.tag} nick={_nick_of(rec)} {exc.code} {exc.message}", + ) from exc + want_id = hid_to_nick.get(rec.hid) or rec.fields.get("id") or "" + tag_def = ss.tag_map.get(rec.tag) + if tag_def: + keys = list(tag_def.fields) + else: + keys = ["id"] + [k for k in rec.fields if k != "id"] + for key in keys: + raw = rec.fields.get(key, "") + if key == "id": + expected = want_id + elif rec.tag == "EDG" and key in ("src", "dist"): + expected = hid_to_nick.get(raw, raw) + else: + expected = raw + got = parsed.fields.get(key, "") + if got != expected: + raise MemNetError( + "snapshot_unsaveable", + f"{rec.tag} nick={_nick_of(rec)} field={key}", + ) + leftover = list(rec_iter) + if leftover: + raise MemNetError( + "snapshot_unsaveable", + f"extra emit rows {len(leftover)}", + ) + + def snapshot_locator_schema_warnings(ss: SessionStore) -> list[str]: """Warn when RAM locator keys will not appear on SCHEMA-shaped snapshot emit. @@ -111,7 +181,9 @@ def snapshot_locator_schema_warnings(ss: SessionStore) -> list[str]: def write_snapshot(ss: SessionStore, path: str | Path) -> int: for msg in snapshot_locator_schema_warnings(ss): emit_wrn("snapshot_schema_drop", msg) - text = snapshot_text(ss) + rec_lines, hid_to_nick = _emit_record_lines(ss) + _verify_emitted_rows(ss, rec_lines, hid_to_nick) + text = _format_snapshot(ss, rec_lines) Path(path).write_text(text, encoding="utf-8") return ss.store.row_count_non_law() diff --git a/parts/common/memnet/memnet/tag_map.py b/parts/common/memnet/memnet/tag_map.py index d483ca6..d0e5154 100644 --- a/parts/common/memnet/memnet/tag_map.py +++ b/parts/common/memnet/memnet/tag_map.py @@ -226,6 +226,21 @@ def _coerce_edg_values(values: list[str], nfields: int) -> list[str]: return values +def value_utf8_len(val: str) -> int: + """Decoded (raw) UTF-8 byte length of a property value (MN-REQ-05.3).""" + return len(val.encode("utf-8")) + + +def check_value_bytes(val: str, caps: Caps) -> None: + """Hard cap on decoded property values. Shared by pipe, GQL, and snapshot load.""" + n = value_utf8_len(val) + if n > caps.max_value_bytes: + raise MemNetError( + "limit_exceeded", + f"value_bytes|{n}/{caps.max_value_bytes}", + ) + + def validate_values(tag_def: TagDef, values: list[str], caps: Caps) -> dict[str, str]: if tag_def.tag == "EDG": values = _coerce_edg_values(values, len(tag_def.fields)) @@ -241,16 +256,8 @@ def validate_values(tag_def: TagDef, values: list[str], caps: Caps) -> dict[str, ) result: dict[str, str] = {} for name, val in zip(tag_def.fields, values, strict=True): - if "\n" in val or "\r" in val: - raise MemNetError( - "newline_in_value", - "newline in field split into two records", - ) - if len(val.encode("utf-8")) > caps.max_value_bytes: - raise MemNetError( - "limit_exceeded", - f"value_bytes|{len(val.encode('utf-8'))}/{caps.max_value_bytes}", - ) + # Newlines are legal once escaped on the snapshot/pipe wire. + check_value_bytes(val, caps) result[name] = val if result.get("id"): validate_id(result["id"]) @@ -272,10 +279,13 @@ def parse_line( used_nicks: set[str] | None = None, ) -> Record: caps = caps or Caps() - if len(line.encode("utf-8")) > caps.max_line_bytes: + physical = len(line.encode("utf-8")) + # Escaped wire may exceed max_line_bytes; decoded size is the documented cap. + physical_hard = max(8 * 1024 * 1024, caps.max_line_bytes) + if physical > physical_hard: raise MemNetError( "limit_exceeded", - f"line_bytes|{len(line.encode('utf-8'))}/{caps.max_line_bytes}", + f"line_bytes|{physical}/{caps.max_line_bytes}", ) try: tag, payload = parse_tag_line(line.strip()) @@ -288,6 +298,13 @@ def parse_line( known = ",".join(tag_map.tag_names()) raise MemNetError("unknown_tag", f"{tag} not in tagMap known: {known}") values = split_payload(payload) + decoded = len(f"@{tag}: ".encode()) + sum(len(v.encode()) for v in values) + decoded += max(0, len(values) - 1) + if decoded > caps.max_line_bytes: + raise MemNetError( + "limit_exceeded", + f"line_bytes|{decoded}/{caps.max_line_bytes}", + ) fields = validate_values(tag_def, values, caps) nick = fields.get("id", "") if nick: diff --git a/parts/common/memnet/memnet/tier_a.py b/parts/common/memnet/memnet/tier_a.py index 4153c09..7b551f7 100644 --- a/parts/common/memnet/memnet/tier_a.py +++ b/parts/common/memnet/memnet/tier_a.py @@ -45,6 +45,7 @@ class NodeRec: same_thing: bool = False absorb_kind: str = "" absorb_match_props: dict[str, str] = field(default_factory=dict) + where: object | None = None @dataclass diff --git a/parts/common/memnet/memnet/wire.py b/parts/common/memnet/memnet/wire.py index 1c53f16..3aac5f8 100644 --- a/parts/common/memnet/memnet/wire.py +++ b/parts/common/memnet/memnet/wire.py @@ -19,6 +19,14 @@ def split_payload(payload: str) -> list[str]: current.append(nxt) i += 2 continue + if nxt == "n": + current.append("\n") + i += 2 + continue + if nxt == "r": + current.append("\r") + i += 2 + continue if ch == "|": fields.append("".join(current)) current = [] @@ -33,7 +41,12 @@ def split_payload(payload: str) -> list[str]: def join_payload(fields: list[str]) -> str: out: list[str] = [] for field in fields: - escaped = field.replace("\\", "\\\\").replace("|", "\\|") + escaped = ( + field.replace("\\", "\\\\") + .replace("|", "\\|") + .replace("\n", "\\n") + .replace("\r", "\\r") + ) out.append(escaped) return "|".join(out) diff --git a/parts/memnet-mcp/software/memnet_mcp/server.py b/parts/memnet-mcp/software/memnet_mcp/server.py index 139767f..d6a7231 100644 --- a/parts/memnet-mcp/software/memnet_mcp/server.py +++ b/parts/memnet-mcp/software/memnet_mcp/server.py @@ -146,9 +146,12 @@ async def session_list() -> str: @mcp.tool() -async def session_close(session: str) -> str: +async def session_close(session: str, caller: str | None = None) -> str: """Close that session id (SessionLifecycle; does not dump S).""" - return await _run(["session", "close", session]) + argv = ["session", "close", session] + if caller: + argv.extend(["--caller", caller]) + return await _run(argv) @mcp.tool() @@ -202,6 +205,7 @@ async def session_load( keep_id: bool = True, ttl: int | None = None, session: str | None = None, + caller: str | None = None, ) -> str: """Load a snapshot file into the MemNet graph (restores session state). @@ -223,6 +227,8 @@ async def session_load( argv.append("--keep-id") if ttl is not None: argv.extend(["--ttl", str(ttl)]) + if caller: + argv.extend(["--caller", caller]) resp = await anyio.to_thread.run_sync(lambda: run_memnet(argv)) return _json(resp) @@ -231,13 +237,17 @@ async def session_load( async def session_save( file: str, session: str | None = None, + caller: str | None = None, ) -> str: """Write the current session graph to a snapshot file. After TTL, only if ``MEMNET_SAVE_ON_EXPIRE`` (then the id is dropped). Auto-dir: ``MEMNET_EXPIRE_SNAPSHOT_DIR``. Not Neo4j. """ - return await _run(["session", "save", "--file", file], session=session) + argv = ["session", "save", "--file", file] + if caller: + argv.extend(["--caller", caller]) + return await _run(argv, session=session) async def _pin_map( From 6a185997b5070a70c5be1ba66178cb3410d4f610 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 13:54:51 +0000 Subject: [PATCH 03/12] Test snapshot round-trip, WHERE, ACL who-check; update cap-contract. Pytest covers lossless specials, 0.19.18 fixture load, GQL value_bytes, unsupported_predicate, fail-closed save, and session save/load/close ACL. Cap-contract probe hits the new wires; bug 4 (GQL mutate skipping the value cap) is removed. Co-authored-by: chouswei --- CHANGELOG.md | 3 + docs/cap-contract.md | 31 ++- tests/cap_contract_lib.py | 150 +++++++++++- tests/fixtures/snapshot-0.19.18.snap | 57 +++++ tests/test_cap_contract.py | 22 ++ tests/test_engine_roundtrip_where_acl.py | 298 +++++++++++++++++++++++ tests/test_sysml_engine_bugs.py | 39 +++ 7 files changed, 586 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/snapshot-0.19.18.snap create mode 100644 tests/test_engine_roundtrip_where_acl.py create mode 100644 tests/test_sysml_engine_bugs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ab6e633..4dfbef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ### Added - **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe and tests; no engine or cap-default change. No SemVer bump. Wire: [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md). +### Fixed +- **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes LF/CR/`|`/`\`; load decodes; save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot round-trip writes no file and RAM still drops (`snap_missing`). GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. + ### Changed - **Invent only — ClusterRoute vs SliceHandCarry (#191 / #47 cousin)** — `MemNetTwoMoves` outside `MemNetSystem` (`MN-REQ-06.9` + `MN-REQ-06.10` / `MN-VER-06-S08`). ClusterRoute = where the session lives (`MemNetLanMcpFront`; one owner; `pin_map` / `find` SHALL NOT span backends). SliceHandCarry = explicit copy into another session (`export_pin_map` or `session_save` → LAN file copy → dest import/`session_load`; `import_slice` same-serve only). Not a live hop. `import_slice(from_url)` not shipped. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/cluster-route-vs-slice-hand-carry.md`](docs/operations/cluster-route-vs-slice-hand-carry.md). - **Invent only — LAN MCP front over several serves (#191)** — `MemNetLanMcpFront` outside `MemNetSystem` (`MN-REQ-06.9` / `MN-VER-06-S07`). One MCP catalogue, N LAN `memnet serve` backends; `SessionOwnerRegistry` is owner (explicit pin allowed; silent hash is not sole routing). One owner per session; `pin_map` / `find` SHALL NOT span backends. Cousin of #47 (peer sid handoff), not the same invent. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/memnet-lan-mcp-front.md`](docs/operations/memnet-lan-mcp-front.md). diff --git a/docs/cap-contract.md b/docs/cap-contract.md index 8402e21..7a72674 100644 --- a/docs/cap-contract.md +++ b/docs/cap-contract.md @@ -151,9 +151,13 @@ Raised at map load (`memnet/tag_map.py`). Session open fails; nothing is stored. | Limit | Default | Knob | Wire | |-------|---------|------|------| | Field value | 4096 | `MEMNET_MAX_VALUE_BYTES` | `@ERR: limit_exceeded\|value_bytes {n}/{max}` | -| Whole pipe line | 32768 | `MEMNET_MAX_LINE_BYTES` | `@ERR: limit_exceeded\|line_bytes {n}/{max}` | +| Whole pipe / snapshot line | 32768 | `MEMNET_MAX_LINE_BYTES` | `@ERR: limit_exceeded\|line_bytes {n}/{max}` | -Enforced on leftover `@TAG` `parse_line` only. **GQL `mutate` does not check these** (bug list). +**Value cap** is one hard cap on leftover `@TAG` `parse_line`, GQL `mutate` (CREATE / SET field values), and snapshot load. It is measured as the **UTF-8 byte length of the decoded (raw) property value**, not the escaped wire form. A product MAY raise `MEMNET_MAX_VALUE_BYTES` to `16384`. Over-cap refuses `limit_exceeded|value_bytes {n}/{max}` and does not store the row. + +**Line cap** is the decoded line size (tag prefix + decoded fields + `|` separators). Snapshot emit escapes LF / CR / `|` / `\` so a value under the value cap cannot trip `line_bytes` on load because of escaping. GQL statements are not pipe lines. + +Snapshot save verifies every emitted row can parse back to the same values. If any row cannot, save refuses `@ERR: snapshot_unsaveable|{tag} nick={nick} …` and writes no file. Expire-save in that case emits `@WRN: expire_snapshot_failed|snapshot_unsaveable`, writes no file, then RAM still drops (`session_expired|snap_missing`). Snapshots written by 0.19.18 (pipe and backslash escapes only) still load. ## Mutate batch line cap @@ -287,8 +291,10 @@ Source: `memnet/catalog_snap.py` `snap_model` / `_precheck_plan`; `memnet/serve. | Expire, save off (default) | Session dropped from memory. First access of the still-registered expired id: `@ERR: session_expired\|snap_missing` (exit 2). After purge already ran: `@ERR: session_not_found\|unknown session` | | Expire, `MEMNET_SAVE_ON_EXPIRE` truthy + `MEMNET_EXPIRE_SNAPSHOT_DIR` set | Snapshot `{dir}/{sid}.snap` (filename only; do not log it). Next use: `@ERR: session_expired\|snap_available`. Restore: `session_load` with that id | | Save-on-expire on, dir unset | `@WRN: save_on_expire_no_dir\|dir unset`, then drop; `snap_missing` | +| Save-on-expire on, row not round-trippable | `@WRN: expire_snapshot_failed\|snapshot_unsaveable`, **no file**, then drop; `snap_missing` | | `session save` after TTL with save-on-expire | Allowed; `@WRN: session_expired_saved`. Id then gone | | `session save` after TTL with save off | `@ERR: session_expired` / `snap_missing`; no file | +| Unsaveable explicit save | `@ERR: snapshot_unsaveable\|{tag} nick={nick} …`; no file | | Status (no paths, no ids) | `@STAT: save_on_expire\|0\|` / `1`; `@STAT: expire_snapshot_dir_set\|0\|` / `1` | Source: `memnet/session.py`, `memnet/config.py` `save_on_expire` / `expire_snapshot_dir`. @@ -308,6 +314,8 @@ Off until `session acl-enable`, a grant/bind (which enables), or `MEMNET_ACL=1` | `acl_scope` | WorkerWriteScope miss | `@ERR: acl_scope\|id/label outside WorkerWriteScope (GRANT)` or `edge outside …` | | `acl_bad_caller` / `acl_bad_bind` / `acl_bad_scope` | Malformed grant/bind/scope | matching `@ERR:` | +When session ACL is enabled, **`session save` / `session load` (into that ACL'd session) / `session close`** accept `--caller` / `MEMNET_CALLER` and enforce `acl_who` / `acl_denied` / `acl_forbidden` (save and load use `pin_map`; close uses `mutate`). MCP `session_save` / `session_load` / `session_close` pass `caller` through. Without ACL, behaviour is unchanged. + Bind is **skipped** when `require_bind=False` and the trusted path is on (`MEMNET_SERVE_INTERNAL` or `MEMNET_TEST_INLINE` or `MEMNET_ACL_SKIP_BIND`). **`memnet serve` sets `MEMNET_SERVE_INTERNAL=1`**, so CLI/MCP through serve does **not** enforce bind today (who and scope still do). Library `MutateGate.apply(..., require_bind=True)` still refuses. Listed as a bug; not fixed in this run. In-scope writes from earlier batches stay; a later out-of-scope batch is refused (not a partial of that batch). @@ -383,17 +391,24 @@ Default `MEMNET_SAVE_ON_EXPIRE` is off. At TTL: 3. Caller sees `@ERR: session_expired|snap_missing` (or `session_not_found` if the id was never known) 4. No snapshot file unless save-on-expire **and** a dir were armed **before** expiry +## MATCH WHERE (honour or refuse) + +GQL `MATCH … WHERE … SET` / `DELETE` SHALL honour the WHERE predicate. Honoured forms: `true` / `false`, property equality / `<>` / `!=`, `CONTAINS`, `STARTS WITH`, `ENDS WITH`, `=~`, `'k' IN n.list`, and `AND` / `OR` / `NOT` of those. Property-map equality in MATCH still filters. `MATCH (n WHERE true)-[r {id}]->() DELETE r` remains the gated edge-delete spelling. + +If lowering cannot honour the predicate, the whole statement refuses and applies nothing: + +`@ERR: unsupported_predicate|WHERE {name} is not honoured` + ## Bugs found this run (do not fix here) 1. **leftover `query walk`** (`WalkQuery` / `context_walk_hops`): clips hops at `max_rows` and fan-out **without** Truncation/`@ERR`. 2. **leftover `query context`**: `context_pack` without `clip_notes` slices `max_rows` silently. 3. **leftover `query neighbors` / `query path`**: depth `min` without Truncation; path may return empty. -4. **GQL `mutate`** does not enforce `MEMNET_MAX_VALUE_BYTES` / `MEMNET_MAX_LINE_BYTES` (pipe leftover does). -5. **`MEMNET_LOCK_TIMEOUT_MS`** is stored on `Caps` and never applied. -6. **`@WRN` budget**: lines after 12 vanish with no mark. -7. **Serve bind skip:** `memnet serve` sets `MEMNET_SERVE_INTERNAL=1`, so session **bind** is not enforced on the TCP/IPC product path (who/scope are). Library `require_bind=True` still refuses. -8. **Serve response frame** over cap raises `ConnectionError` rather than `@ERR: frame_too_large`. -9. **`max_fanout`** clamps only outgoing `_edges_from`, not inbound `_edges_to`, so a high in-degree hub is not Truncation-marked for fan-out. +4. **`MEMNET_LOCK_TIMEOUT_MS`** is stored on `Caps` and never applied. +5. **`@WRN` budget**: lines after 12 vanish with no mark. +6. **Serve bind skip:** `memnet serve` sets `MEMNET_SERVE_INTERNAL=1`, so session **bind** is not enforced on the TCP/IPC product path (who/scope are). Library `require_bind=True` still refuses. +7. **Serve response frame** over cap raises `ConnectionError` rather than `@ERR: frame_too_large`. +8. **`max_fanout`** clamps only outgoing `_edges_from`, not inbound `_edges_to`, so a high in-degree hub is not Truncation-marked for fan-out. Product `pin_map` Truncation for `max_rows` / `depth` / `fanout` / `shell` **is** signalled. Mutate/ingest row-cap batches **do** roll back. diff --git a/tests/cap_contract_lib.py b/tests/cap_contract_lib.py index 30257c2..8482a2b 100644 --- a/tests/cap_contract_lib.py +++ b/tests/cap_contract_lib.py @@ -38,6 +38,7 @@ reset_registry, set_now_override, ) +from memnet.snapshot import write_snapshot from memnet.tag_map import load_map_from_lines, load_user_map, parse_line from memnet.walk_query import WalkQuery @@ -473,12 +474,32 @@ def case_pipe_value_and_line_bytes() -> list[Case]: name="pipe_value_bytes", kind="hard_refuse", default="4096", - knob="MEMNET_MAX_VALUE_BYTES (pipe parse_line only)", + knob="MEMNET_MAX_VALUE_BYTES", library_code=exc.code, library_message=exc.message, wire=format_err(exc.code, exc.message), - extra={"gql_mutate": "does not enforce this cap"}, - bug="GQL mutate does not check max_value_bytes", + extra={"measured_on": "decoded raw UTF-8"}, + ) + ) + with env_caps(MEMNET_MAX_VALUE_BYTES="4"): + ss = _open() + try: + MutateGate(ss).apply( + ["CREATE (:CST {id: 'N09', name: 'toolong', role: 'x'})"], + mode="add", + ) + raise AssertionError("expected gql value_bytes") + except MemNetError as exc: + out.append( + Case( + name="gql_mutate_value_bytes", + kind="hard_refuse", + default="4096", + knob="MEMNET_MAX_VALUE_BYTES", + library_code=exc.code, + library_message=exc.message, + wire=format_err(exc.code, exc.message), + extra={"measured_on": "decoded raw UTF-8"}, ) ) with env_caps(MEMNET_MAX_LINE_BYTES="8"): @@ -492,12 +513,11 @@ def case_pipe_value_and_line_bytes() -> list[Case]: name="pipe_line_bytes", kind="hard_refuse", default="32768", - knob="MEMNET_MAX_LINE_BYTES (pipe parse_line only)", + knob="MEMNET_MAX_LINE_BYTES (decoded line; leftover pipe / snapshot)", library_code=exc.code, library_message=exc.message, wire=format_err(exc.code, exc.message), - extra={"gql_mutate": "does not enforce this cap"}, - bug="GQL mutate does not check max_line_bytes", + extra={"gql_mutate": "GQL statements are not pipe lines"}, ) ) return out @@ -930,6 +950,123 @@ def case_acl() -> list[Case]: extra={"partial": "first in-scope write stayed; second refused (separate batches)"}, ) ) + ss3 = _open() + ss3.grant_caller("owner", can_pin_map=True, can_mutate=True) + denied_save = _cli( + ["session", "save", "--file", str(Path("/tmp/acl-save.snap")), "--session", ss3.session_id] + ) + out.append( + Case( + name="acl_who_session_save", + kind="hard_refuse", + default="ACL off until session acl-enable / grant", + knob="session save --caller / MEMNET_CALLER", + library_code="acl_who", + library_message="caller id required when session ACL is enabled", + wire=redact(denied_save.stderr or ""), + extra={"exit_code": str(denied_save.exit_code)}, + ) + ) + denied_close = _cli(["session", "close", ss3.session_id]) + out.append( + Case( + name="acl_who_session_close", + kind="hard_refuse", + default="ACL off until session acl-enable / grant", + knob="session close --caller / MEMNET_CALLER", + library_code="acl_who", + library_message="caller id required when session ACL is enabled", + wire=redact(denied_close.stderr or ""), + extra={"exit_code": str(denied_close.exit_code)}, + ) + ) + denied_load = _cli(["session", "load", "--session", ss3.session_id]) + out.append( + Case( + name="acl_who_session_load", + kind="hard_refuse", + default="ACL off until session acl-enable / grant", + knob="session load --caller / MEMNET_CALLER", + library_code="acl_who", + library_message="caller id required when session ACL is enabled", + wire=redact(denied_load.stderr or ""), + extra={"exit_code": str(denied_load.exit_code)}, + ) + ) + return out + + +def case_where_and_snapshot_honesty() -> list[Case]: + """MN-REQ-03.4 / MN-REQ-01.9 — WHERE honour-or-refuse; fail-closed snapshot.""" + _clean() + out: list[Case] = [] + ss = _open() + MutateGate(ss).apply([_cst(1)], mode="add") + try: + MutateGate(ss).apply( + ["MATCH (n:CST) WHERE n.role > 0 SET n.role = 'x'"], + mode="mutate", + ) + raise AssertionError("expected unsupported_predicate") + except MemNetError as exc: + out.append( + Case( + name="unsupported_predicate", + kind="hard_refuse", + default="honour WHERE or refuse", + knob="GQL MATCH WHERE SET/DELETE", + library_code=exc.code, + library_message=exc.message, + wire=format_err(exc.code, exc.message), + extra={ + "applied": "false", + "wealth_or_role": ss.store.get("N01").fields.get("role", ""), + }, + ) + ) + try: + MutateGate(ss).apply( + ["MATCH (n:CST {id: 'N01'}) WHERE n.name CONTAINS 'nope' SET n.role = 'x'"], + mode="mutate", + ) + raise AssertionError("expected not_found") + except MemNetError as exc: + out.append( + Case( + name="where_false_unique_set", + kind="hard_refuse", + default="WHERE filters unique MATCH", + knob="GQL MATCH WHERE SET", + library_code=exc.code, + library_message=exc.message, + wire=format_err(exc.code, exc.message), + extra={"role_unchanged": ss.store.get("N01").fields.get("role", "")}, + ) + ) + ss2 = _open() + MutateGate(ss2).apply([_cst(1, name="n1")], mode="add") + rec = ss2.store.get("N01") + rec.fields["name"] = "x" * 9000 + try: + write_snapshot(ss2, Path("/tmp/unsaveable.snap")) + raise AssertionError("expected snapshot_unsaveable") + except MemNetError as exc: + out.append( + Case( + name="snapshot_unsaveable", + kind="hard_refuse", + default="save fail-closed", + knob="session save / write_snapshot", + library_code=exc.code, + library_message=exc.message, + wire=format_err(exc.code, exc.message), + extra={ + "expire": ( + "@WRN: expire_snapshot_failed|snapshot_unsaveable then snap_missing" + ), + }, + ) + ) return out @@ -1113,6 +1250,7 @@ def collect_all(tmp_path: Path) -> list[Case]: case_snap_session_precheck(tmp_path), *case_ttl_and_expire(tmp_path), *case_acl(), + *case_where_and_snapshot_honesty(), case_reserve(), case_slice_budget(), case_frame_too_large(), diff --git a/tests/fixtures/snapshot-0.19.18.snap b/tests/fixtures/snapshot-0.19.18.snap new file mode 100644 index 0000000..42a79c9 --- /dev/null +++ b/tests/fixtures/snapshot-0.19.18.snap @@ -0,0 +1,57 @@ +# memnet-snapshot-v1 +@SNAP: 1|snap_legacy_sid|2026-10-08T13:29:27.826384Z|2026-10-08T14:29:27.826384Z|60|1|2026-10-08T13:29:27.839687Z +# map +SCHEMA BIZ ; fields=id name type location profit cashflow employees recycle +SCHEMA CFG ; fields=id world economy identity core_ability crisis +SCHEMA EDG ; fields=id src relation dist at attrs recycle +SCHEMA LAW ; fields=id name cycle mechanism constraint +SCHEMA NPC ; fields=id name traits corruption craft funding_gap status recycle +SCHEMA PLR ; fields=id identity wealth cashflow monopoly reputation inventory +SCHEMA PRD ; fields=id name type cost price status +SCHEMA SYS ; fields=id round time deficit revenue chaos exchange_rate +SCHEMA TEC ; fields=id name domain status effect +SCHEMA TSK ; fields=id goal deadline status recycle +# relations +@REL: allocates +@REL: bind +@REL: binds +@REL: calls +@REL: connects +@REL: constrained_by +@REL: contains +@REL: declaredIn +@REL: defines +@REL: delegates +@REL: dependsOn +@REL: documents +@REL: flowOf +@REL: governs +@REL: hasPort +@REL: helps +@REL: implements +@REL: inFile +@REL: includes +@REL: knows +@REL: links +@REL: maps_to +@REL: memberOf +@REL: member_of +@REL: mentions +@REL: next +@REL: on_net +@REL: overrides +@REL: owns +@REL: paired_with +@REL: pipe +@REL: preempts +@REL: produces +@REL: realizes +@REL: reports_to +@REL: satisfies +@REL: seeks_help +@REL: tests +@REL: triggers +@REL: typedBy +@REL: uses +# records +@PLR: PLR_V118|pipe\|slash\\q {x} $ cjk测例|1|0|0|0|bag diff --git a/tests/test_cap_contract.py b/tests/test_cap_contract.py index 438f685..b415e4f 100644 --- a/tests/test_cap_contract.py +++ b/tests/test_cap_contract.py @@ -102,8 +102,27 @@ def test_every_cap_and_write_proof(memnet_temp, tmp_path: Path): val = by_name["pipe_value_bytes"] assert val.library_code == "limit_exceeded" assert val.wire.startswith("@ERR: limit_exceeded|value_bytes") + assert not val.bug + gql_val = by_name["gql_mutate_value_bytes"] + assert gql_val.library_code == "limit_exceeded" + assert gql_val.wire.startswith("@ERR: limit_exceeded|value_bytes") + assert not gql_val.bug + unpred = by_name["unsupported_predicate"] + assert unpred.library_code == "unsupported_predicate" + assert unpred.wire.startswith("@ERR: unsupported_predicate|WHERE") + assert "is not honoured" in unpred.wire + unsave = by_name["snapshot_unsaveable"] + assert unsave.library_code == "snapshot_unsaveable" + assert unsave.wire.startswith("@ERR: snapshot_unsaveable|") + who_save = by_name["acl_who_session_save"] + assert "acl_who" in who_save.wire + who_close = by_name["acl_who_session_close"] + assert "acl_who" in who_close.wire + who_load = by_name["acl_who_session_load"] + assert "acl_who" in who_load.wire lineb = by_name["pipe_line_bytes"] assert lineb.wire.startswith("@ERR: limit_exceeded|line_bytes") + assert not lineb.bug batch = by_name["mutate_batch_lines"] assert batch.wire == "@ERR: limit_exceeded|batch_lines 3/2" @@ -227,6 +246,9 @@ def test_doc_lists_live_defaults_and_wires(): "serve_timeout", "snap_model", "1 catalog + N interiors", + "MEMNET_MAX_VALUE_BYTES", + "unsupported_predicate", + "snapshot_unsaveable", ): assert needle in text, needle diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py new file mode 100644 index 0000000..4c9ed73 --- /dev/null +++ b/tests/test_engine_roundtrip_where_acl.py @@ -0,0 +1,298 @@ +"""MN-REQ-01.9 / 01.10 / 03.4 / 05.3 — snapshot round-trip, WHERE, ACL who.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from memnet.cli import app +from memnet.config import Caps +from memnet.exceptions import MemNetError +from memnet.mutate_gate import MutateGate +from memnet.session import get_session, open_session, snapshot_expired_session +from memnet.snapshot import load_snapshot, write_snapshot +from memnet.wire import join_payload, split_payload + +runner = CliRunner() +FIXTURE = Path(__file__).parent / "fixtures" / "snapshot-0.19.18.snap" + +_PLR = ( + "CREATE (:PLR {id: 'PLR01', identity: 'Hero', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" +) + + +def _gql_escape(val: str) -> str: + return val.replace("\\", "\\\\").replace("'", "\\'").replace("\n", "\\n").replace("\r", "\\r") + + +def test_split_join_newlines_and_specials(): + raw = "a|b\nc\\d\r测例$\"'{x}" + wire = join_payload([raw]) + assert "\n" not in wire and "\r" not in wire + assert split_payload(wire) == [raw] + + +def test_legacy_0_19_18_snapshot_loads(memnet_temp): + ss = load_snapshot(FIXTURE) + rec = ss.store.get("PLR_V118") + assert rec is not None + assert rec.fields["identity"] == r"pipe|slash\q {x} $ cjk测例" + + +def test_snapshot_roundtrip_specials_and_newlines(memnet_temp, schema_file, tmp_path: Path): + ss = open_session(map_file=str(schema_file)) + blob = "line1\nline2\r" + r"""slash\ quotes"' $ {brace} |pipe| 测例""" + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR_RT', identity: '" + + _gql_escape(blob) + + "', wealth: 1, cashflow: 0, monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + path = tmp_path / "rt.snap" + write_snapshot(ss, path) + text = path.read_text(encoding="utf-8") + assert "\n@PLR:" in text or text.splitlines()[0] == "# memnet-snapshot-v1" + rec_line = next(ln for ln in text.splitlines() if ln.startswith("@PLR:")) + assert "\n" not in rec_line[1:] or rec_line.count("@PLR:") == 1 + loaded = load_snapshot(path) + got = loaded.store.get("PLR_RT") + assert got is not None + assert got.fields["identity"] == blob + + +def test_gql_mutate_value_bytes_hard_cap(memnet_temp, schema_file, monkeypatch): + monkeypatch.setenv("MEMNET_MAX_VALUE_BYTES", "8") + ss = open_session(map_file=str(schema_file), caps=Caps()) + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR_BIG', identity: 'toolongval', wealth: 1, " + "cashflow: 0, monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + assert ei.value.code == "limit_exceeded" + assert ei.value.message.startswith("value_bytes|") + assert ss.store.get("PLR_BIG") is None + + +def test_escaped_under_value_cap_does_not_trip_line_bytes( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + monkeypatch.setenv("MEMNET_MAX_VALUE_BYTES", "64") + monkeypatch.setenv("MEMNET_MAX_LINE_BYTES", "80") + ss = open_session(map_file=str(schema_file), caps=Caps()) + blob = "n" * 20 + "\n" * 20 + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR_ESC', identity: '" + + _gql_escape(blob) + + "', wealth: 1, cashflow: 0, monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + path = tmp_path / "esc.snap" + write_snapshot(ss, path) + loaded = load_snapshot(path, caps=Caps()) + assert loaded.store.get("PLR_ESC").fields["identity"] == blob + + +def test_where_contains_filters_and_false_set_is_noop(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR02', identity: 'Villain', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity CONTAINS 'nope' SET n.wealth = 9"], + mode="mutate", + ) + assert ei.value.code == "not_found" + assert ss.store.get("PLR01").fields["wealth"] == "1" + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity CONTAINS 'Hero' SET n.wealth = 9"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["wealth"] == "9" + assert ss.store.get("PLR02").fields["wealth"] == "1" + + +def test_where_equality_starts_ends_in_and_regex(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity = 'Hero' SET n.cashflow = 2"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["cashflow"] == "2" + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity STARTS WITH 'He' SET n.monopoly = 3"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["monopoly"] == "3" + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity ENDS WITH 'ro' SET n.reputation = 4"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["reputation"] == "4" + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity =~ 'H.*o' SET n.inventory = 'box'"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["inventory"] == "box" + MutateGate(ss).apply( + ["MATCH (n:PLR {id: 'PLR01'}) SET n.inventory = '[\"bag\",\"key\"]'"], + mode="mutate", + ) + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE 'key' IN n.inventory SET n.wealth = 8"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["wealth"] == "8" + + +def test_where_false_unique_match_does_not_set(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + ["MATCH (n:PLR {id: 'PLR01'}) WHERE n.identity CONTAINS 'zzz' SET n.wealth = 9"], + mode="mutate", + ) + assert ei.value.code == "not_found" + assert ss.store.get("PLR01").fields["wealth"] == "1" + + +def test_where_false_delete_is_noop(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.identity CONTAINS 'zzz' DELETE n"], + mode="mutate", + ) + assert ei.value.code == "not_found" + assert ss.store.get("PLR01") is not None + + +def test_unsupported_where_refuses_nothing_applied(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + with pytest.raises(MemNetError) as ei: + MutateGate(ss).apply( + ["MATCH (n:PLR) WHERE n.wealth > 0 SET n.wealth = 9"], + mode="mutate", + ) + assert ei.value.code == "unsupported_predicate" + assert "WHERE" in ei.value.message + assert ss.store.get("PLR01").fields["wealth"] == "1" + + +def test_where_true_edge_delete_still_works(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR02', identity: 'Villain', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + MutateGate(ss).apply( + ["MATCH (a {id: 'PLR01'}), (b {id: 'PLR02'}) CREATE (a)-[:knows {id: 'E_k'}]->(b)"], + mode="add", + allow_new_relation=True, + ) + MutateGate(ss).apply( + ["MATCH (n WHERE true)-[r {id: 'E_k'}]->() DELETE r"], + mode="mutate", + ) + assert ss.store.get("E_k") is None + + +def test_map_equality_still_filters(memnet_temp, schema_file): + ss = open_session(map_file=str(schema_file)) + MutateGate(ss).apply([_PLR], mode="add") + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR02', identity: 'Villain', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + MutateGate(ss).apply( + ["MATCH (n:PLR {id: 'PLR02'}) SET n.wealth = 7"], + mode="mutate", + ) + assert ss.store.get("PLR01").fields["wealth"] == "1" + assert ss.store.get("PLR02").fields["wealth"] == "7" + + +def test_cli_acl_save_load_close_who(memnet_temp, schema_file, tmp_path: Path): + r1 = runner.invoke(app, ["session", "open", "--map-file", str(schema_file)]) + sid = r1.stdout.strip().split("|")[0].replace("@SESSION: ", "") + runner.invoke(app, ["session", "acl-enable", "--session", sid]) + runner.invoke( + app, + ["session", "acl-grant", "--caller", "owner", "--session", sid], + ) + snap = tmp_path / "acl.snap" + denied = runner.invoke(app, ["session", "save", "--file", str(snap), "--session", sid]) + assert denied.exit_code != 0 + assert "acl_who" in denied.stderr + wrong = runner.invoke( + app, + ["session", "save", "--file", str(snap), "--session", sid, "--caller", "intruder"], + ) + assert "acl_denied" in wrong.stderr + ok = runner.invoke( + app, + ["session", "save", "--file", str(snap), "--session", sid, "--caller", "owner"], + ) + assert ok.exit_code == 0, ok.stderr + close_no = runner.invoke(app, ["session", "close", sid]) + assert "acl_who" in close_no.stderr + assert get_session(sid).session_id == sid + load_no = runner.invoke(app, ["session", "load", "--session", sid]) + assert "acl_who" in load_no.stderr + closed = runner.invoke(app, ["session", "close", sid, "--caller", "owner"]) + assert closed.exit_code == 0, closed.stderr + + +def test_explicit_save_unsaveable_names_row(memnet_temp, schema_file, tmp_path: Path): + ss = open_session(map_file=str(schema_file), caps=Caps()) + MutateGate(ss).apply([_PLR], mode="add") + rec = ss.store.get("PLR01") + rec.fields["identity"] = "x" * 9000 + path = tmp_path / "bad.snap" + with pytest.raises(MemNetError) as ei: + write_snapshot(ss, path) + assert ei.value.code == "snapshot_unsaveable" + assert "PLR" in ei.value.message + assert "PLR01" in ei.value.message + assert not path.exists() + + +def test_expire_save_unsaveable_writes_no_file( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + """If round-trip verify fails, expire-save warns and leaves no snap file.""" + monkeypatch.setenv("MEMNET_SAVE_ON_EXPIRE", "1") + monkeypatch.setenv("MEMNET_EXPIRE_SNAPSHOT_DIR", str(tmp_path)) + ss = open_session(map_file=str(schema_file), caps=Caps()) + MutateGate(ss).apply([_PLR], mode="add") + rec = ss.store.get("PLR01") + rec.fields["identity"] = "x" * 9000 + snapshot_expired_session(ss.session_id, Caps()) + snaps = list(tmp_path.glob("*.snap")) + assert snaps == [] diff --git a/tests/test_sysml_engine_bugs.py b/tests/test_sysml_engine_bugs.py new file mode 100644 index 0000000..e42aaea --- /dev/null +++ b/tests/test_sysml_engine_bugs.py @@ -0,0 +1,39 @@ +"""Honesty: MN-REQ-01.9 / 01.10 / 03.4 / 05.3 live in the SysML SSOT.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MODELS = ROOT / "sysml-models" / "models" +REQ = MODELS / "requirements.sysml" +DEPLOY = MODELS / "deploy.sysml" +VERIFY = MODELS / "verify.sysml" + + +def test_requirements_ids_present(): + text = REQ.read_text(encoding="utf-8") + for nid in ("MN-REQ-01.9", "MN-REQ-01.10", "MN-REQ-03.4", "MN-REQ-05.3"): + assert nid in text + assert "snapshot_unsaveable" in text + assert "unsupported_predicate" in text + assert "MEMNET_MAX_VALUE_BYTES" in text + assert "snapshotLosslessRoundTripReq" in text + assert "sessionLifecycleAclWhoReq" in text + assert "honourWherePredicateReq" in text + assert "consistentValueByteCapReq" in text + + +def test_deploy_and_verify_trail_model(): + deploy = DEPLOY.read_text(encoding="utf-8") + assert "losslessPropertyRoundTrip" in deploy + assert "failClosedUnsaveable" in deploy + assert "honourWhereOrRefuse" in deploy + assert 'envMaxValueBytes : String = "MEMNET_MAX_VALUE_BYTES"' in deploy + assert "sessionSaveLoadCloseAclWho" in deploy + assert "acceptsCaller" in deploy + ver = VERIFY.read_text(encoding="utf-8") + assert "MN-VER-01-S04" in ver + assert "MN-VER-01-S05" in ver + assert "MN-VER-03-S01" in ver + assert "MN-VER-05-S01" in ver From 4ef73838d66ffd1d670baf649cd3019bb330ac88 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:17:22 +0000 Subject: [PATCH 04/12] Extend MN-REQ-01.9 and 05.3 for splitlines seps, extras, and line_bytes. Co-authored-by: chouswei --- sysml-models/models/deploy.sysml | 5 ++++ sysml-models/models/requirements.sysml | 38 ++++++++++++++++++-------- sysml-models/models/verify.sysml | 17 ++++++++++-- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index fbb9453..e8b5c42 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -171,6 +171,7 @@ package MemNet { attribute maxValueBytesDefault : Integer = 4096; attribute valueBytesOnDecodedRaw : Boolean = true; attribute gqlMutateEnforcesValueBytes : Boolean = true; + attribute lineBytesOnEmittedLine : Boolean = true; attribute sessionSaveLoadCloseAclWho : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_05_HardCaps::MN_REQ_05_1_StoreResourceCaps; @@ -1090,6 +1091,10 @@ package MemNet { attribute expireSaveUnsaveableNoFile : Boolean = true; attribute legacy01918Load : Boolean = true; attribute escapeLfCrPipeBackslash : Boolean = true; + attribute splitlinesSeparatorsEscaped : Boolean = true; + attribute recordSplitLfOnly : Boolean = true; + attribute persistUndeclaredProperties : Boolean = true; + attribute lineBytesOnEmittedLine : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_4_SaveSessionSnapshot; satisfy MN_REQ_00_MissionBridge::MN_REQ_01_SessionLifecycle::MN_REQ_01_5_LoadSessionSnapshot; diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index fec7c44..da9f358 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -213,14 +213,24 @@ package MemNetRequirements { doc /* Anything the write path accepts SHALL session_save and session_load byte-for-byte as the same property values. - That includes LF, CR, backslash, both quote kinds, dollar, - braces, pipe, and CJK. Snapshot emit SHALL escape those - losslessly; load SHALL decode them. Save SHALL fail closed - (snapshot_unsaveable naming the row) rather than write an - unloadable file. Expire-save that cannot round-trip SHALL - NOT write a file; RAM still drops; next use is - session_expired|snap_missing. Snapshots written by 0.19.18 - (pipe and backslash escapes only) SHALL still load. + That includes LF, CR, tab, VT, FF, file/group/record + separators, NEL, line/paragraph separators, backslash, + both quote kinds, dollar, braces, pipe, and CJK. Snapshot + emit SHALL escape those losslessly; load SHALL decode them. + Record split SHALL be LF-only (MUST NOT use + str.splitlines(), which splits on CR/VT/FF/NEL/LS/PS + before unescape). Properties mutate stores that are absent + from the live tag SCHEMA SHALL persist by widening the + snapshot SCHEMA (GraphElement extras and Path-B locators + such as qname). Live session SCHEMA is unchanged. Fixed + tags EDG/LAW SHALL NOT widen; extras there SHALL fail + closed. Save SHALL fail closed (snapshot_unsaveable + naming the row) rather than write an unloadable file, + including when extras would exceed max_fields or the + escaped line exceeds line_bytes. Expire-save that cannot + round-trip SHALL NOT write a file; RAM still drops; next + use is session_expired|snap_missing. Snapshots written by + 0.19.18 (pipe and backslash escapes only) SHALL still load. */ attribute requirementId : String = "MN-REQ-01.9"; } @@ -660,10 +670,14 @@ package MemNetRequirements { Default 4096. Knob MEMNET_MAX_VALUE_BYTES (a product MAY raise it to 16384). Over-cap SHALL refuse limit_exceeded|value_bytes n/max and SHALL NOT accept the - write. An escaped snapshot value whose decoded size is under - this cap SHALL NOT trip line_bytes on load. line_bytes - remains a leftover-pipe / snapshot decoded-line cap - (MEMNET_MAX_LINE_BYTES, default 32768). + write. line_bytes is the UTF-8 byte length of the + escaped/raw leftover-pipe or snapshot line (backslash and + pipe count twice). Save verify and load SHALL use the same + check (MEMNET_MAX_LINE_BYTES, default 32768). A single + field under value_bytes still fits default line_bytes; + many fields whose escaped form exceeds line_bytes SHALL + fail closed at save (snapshot_unsaveable wrapping + line_bytes) rather than write a file load will refuse. */ attribute requirementId : String = "MN-REQ-05.3"; } diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index f10a828..1ce8c1f 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -1860,8 +1860,11 @@ package MemNetVerification { verification def MN_VER_01_S04_SnapshotLosslessRoundTrip { doc /* - MN-REQ-01.9 / MN-REQ-05.3 — snapshot emit escapes LF/CR/pipe/backslash - losslessly; save fails closed; expire-save writes no unloadable file; + MN-REQ-01.9 / MN-REQ-05.3 — snapshot emit escapes splitlines + separators (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus pipe/backslash + losslessly; undeclared RAM properties persist by snapshot SCHEMA + widen; line_bytes is the escaped emitted line at save and load; + save fails closed; expire-save writes no unloadable file; 0.19.18 snapshots still load; value_bytes is decoded raw UTF-8. */ attribute verificationId : String = "MN-VER-01-S04"; @@ -1882,6 +1885,14 @@ package MemNetVerification { .legacy01918Load == true and system.core.transport.inProcess.memory.sessions.snap .escapeLfCrPipeBackslash == true + and system.core.transport.inProcess.memory.sessions.snap + .splitlinesSeparatorsEscaped == true + and system.core.transport.inProcess.memory.sessions.snap + .recordSplitLfOnly == true + and system.core.transport.inProcess.memory.sessions.snap + .persistUndeclaredProperties == true + and system.core.transport.inProcess.memory.sessions.snap + .lineBytesOnEmittedLine == true and system.core.cli.sessionSave.failClosedUnsaveable == true and system.core.cli.sessionLoad.legacy01918Load == true and system.core.transport.inProcess.memory.sessions.caps @@ -1958,6 +1969,8 @@ package MemNetVerification { .valueBytesOnDecodedRaw == true and system.core.transport.inProcess.memory.sessions.caps .gqlMutateEnforcesValueBytes == true + and system.core.transport.inProcess.memory.sessions.caps + .lineBytesOnEmittedLine == true and system.core.transport.inProcess.memory.sessions.recallCommit .commit.mutate.gqlValueBytesHardCap == true } From 8d3dd66e47b1d2cd88825598742e35170b90e800 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:17:33 +0000 Subject: [PATCH 05/12] Escape splitlines separators, persist undeclared snapshot properties, check line_bytes at save. Co-authored-by: chouswei --- parts/common/memnet/memnet/snapshot.py | 120 +++++++++++++++++-------- parts/common/memnet/memnet/tag_map.py | 13 +-- parts/common/memnet/memnet/wire.py | 76 ++++++++++++++-- 3 files changed, 151 insertions(+), 58 deletions(-) diff --git a/parts/common/memnet/memnet/snapshot.py b/parts/common/memnet/memnet/snapshot.py index c8a756b..5869c58 100644 --- a/parts/common/memnet/memnet/snapshot.py +++ b/parts/common/memnet/memnet/snapshot.py @@ -9,8 +9,9 @@ from memnet.config import Caps from memnet.exceptions import MemNetError +from memnet.fixed_tags import FIXED_TAGS from memnet.mem_store import MemStore -from memnet.models import Record, SessionMeta +from memnet.models import Record, SessionMeta, TagDef, TagMap from memnet.output import emit_record, emit_wrn from memnet.registry import SessionEntry, count, register from memnet.session import SessionStore, purge_expired, utc_now @@ -21,14 +22,12 @@ tag_map_to_lines, validate_id, ) +from memnet.wire import split_snapshot_lines SNAPSHOT_MAGIC = "# memnet-snapshot-v1" _SECTION_MAP = "# map" _SECTION_REL = "# relations" _SECTION_REC = "# records" -# Locator keys agents cue with. emit_record persists SCHEMA columns only; -# extras vanish on session_save unless listed on the map. -LOCATOR_PERSIST_KEYS = frozenset({"qname", "path", "requirementId", "skill_id"}) def _snapshot_emit_nick(rec: Record, used: set[str]) -> str: @@ -47,7 +46,54 @@ def _nick_of(rec: Record) -> str: return rec.fields.get("id") or rec.tag -def _emit_record_lines(ss: SessionStore) -> tuple[list[str], dict[str, str]]: +def _snapshot_emit_tag_map(ss: SessionStore) -> TagMap: + """Widen SCHEMA with undeclared RAM keys so extras round-trip (MN-REQ-01.9).""" + extras: dict[str, list[str]] = {} + for rid in ss.store.write_order: + rec = ss.store._by_hid.get(rid) + if not rec: + continue + td = ss.tag_map.get(rec.tag) + base = list(td.fields) if td else ["id"] + known = set(base) + extra = extras.setdefault(rec.tag, []) + for key in rec.fields: + if key in known: + continue + if rec.tag in FIXED_TAGS: + raise MemNetError( + "snapshot_unsaveable", + f"{rec.tag} nick={_nick_of(rec)} field={key} fixed_tag extra", + ) + extra.append(key) + known.add(key) + tags: dict[str, TagDef] = {} + max_fields = ss.caps.max_fields + for tag, td in ss.tag_map.tags.items(): + fields = list(td.fields) + extras.get(tag, []) + if len(fields) > max_fields: + raise MemNetError( + "snapshot_unsaveable", + f"{tag} nick=- fields|{len(fields)}/{max_fields}", + ) + tags[tag] = TagDef(tag=td.tag, fields=fields, kind=td.kind) + for tag, extra in extras.items(): + if tag in tags or not extra: + continue + fields = ["id"] + extra + if len(fields) > max_fields: + raise MemNetError( + "snapshot_unsaveable", + f"{tag} nick=- fields|{len(fields)}/{max_fields}", + ) + tags[tag] = TagDef(tag=tag, fields=fields, kind="node") + return TagMap(tags=tags) + + +def _emit_record_lines( + ss: SessionStore, +) -> tuple[list[str], dict[str, str], TagMap]: + emit_map = _snapshot_emit_tag_map(ss) used: set[str] = set() hid_to_nick: dict[str, str] = {} for rid in ss.store.write_order: @@ -67,17 +113,18 @@ def _emit_record_lines(ss: SessionStore) -> tuple[list[str], dict[str, str]]: if token in hid_to_nick: fields[key] = hid_to_nick[token] clone = rec.model_copy(update={"fields": fields}) - rec_lines.append(emit_record(clone, ss.tag_map)) - return rec_lines, hid_to_nick + rec_lines.append(emit_record(clone, emit_map)) + return rec_lines, hid_to_nick, emit_map def snapshot_text(ss: SessionStore) -> str: - rec_lines, hid_to_nick = _emit_record_lines(ss) - _verify_emitted_rows(ss, rec_lines, hid_to_nick) - return _format_snapshot(ss, rec_lines) + rec_lines, hid_to_nick, emit_map = _emit_record_lines(ss) + text = _format_snapshot(ss, rec_lines, emit_map) + _verify_emitted_snapshot(ss, text, hid_to_nick, emit_map) + return text -def _format_snapshot(ss: SessionStore, rec_lines: list[str]) -> str: +def _format_snapshot(ss: SessionStore, rec_lines: list[str], emit_map: TagMap) -> str: lines = [SNAPSHOT_MAGIC] m = ss.meta hw = "1" if m.has_writes else "0" @@ -86,7 +133,7 @@ def _format_snapshot(ss: SessionStore, rec_lines: list[str]) -> str: f"@SNAP: 1|{m.session_id}|{m.created_at}|{m.expires_at}|{m.ttl_minutes}|{hw}|{modified}" ) lines.append(_SECTION_MAP) - lines.extend(tag_map_to_lines(ss.tag_map)) + lines.extend(tag_map_to_lines(emit_map)) lines.append(_SECTION_REL) for rel in sorted(ss.relations): lines.append(f"@REL: {rel}") @@ -95,13 +142,15 @@ def _format_snapshot(ss: SessionStore, rec_lines: list[str]) -> str: return "\n".join(lines) + "\n" -def _verify_emitted_rows( +def _verify_emitted_snapshot( ss: SessionStore, - rec_lines: list[str], + text: str, hid_to_nick: dict[str, str], + emit_map: TagMap, ) -> None: - """Refuse save if any emitted row would not load as the same values.""" + """Refuse save if the formatted blob would not load as the same values.""" used_nicks: set[str] = set() + _, _, _, rec_lines = _parse_sections(split_snapshot_lines(text)) rec_iter = iter(rec_lines) for rid in ss.store.write_order: rec = ss.store._by_hid.get(rid) @@ -115,18 +164,16 @@ def _verify_emitted_rows( f"{rec.tag} nick={_nick_of(rec)} missing emit row", ) from exc try: - parsed = parse_line(line, ss.tag_map, ss.caps, used_nicks=used_nicks) + parsed = parse_line(line, emit_map, ss.caps, used_nicks=used_nicks) except MemNetError as exc: raise MemNetError( "snapshot_unsaveable", f"{rec.tag} nick={_nick_of(rec)} {exc.code} {exc.message}", ) from exc want_id = hid_to_nick.get(rec.hid) or rec.fields.get("id") or "" - tag_def = ss.tag_map.get(rec.tag) - if tag_def: - keys = list(tag_def.fields) - else: - keys = ["id"] + [k for k in rec.fields if k != "id"] + keys = list(rec.fields.keys()) + if "id" not in keys: + keys = ["id", *keys] for key in keys: raw = rec.fields.get(key, "") if key == "id": @@ -150,40 +197,35 @@ def _verify_emitted_rows( def snapshot_locator_schema_warnings(ss: SessionStore) -> list[str]: - """Warn when RAM locator keys will not appear on SCHEMA-shaped snapshot emit. + """Extras persist by widening snapshot SCHEMA. Warn only if save cannot. - Does not change SCHEMA. Honesty only: session_save otherwise drops extras - such as Path-B ``qname`` when the map omitted them. + Fixed-tag extras (EDG / LAW) cannot widen; save refuses instead. """ - seen: set[tuple[str, str]] = set() msgs: list[str] = [] + seen: set[tuple[str, str]] = set() for rid in ss.store.write_order: rec = ss.store._by_hid.get(rid) - if not rec: + if not rec or rec.tag not in FIXED_TAGS: continue - tag_def = ss.tag_map.get(rec.tag) - schema_fields = set(tag_def.fields) if tag_def else set() - for key in LOCATOR_PERSIST_KEYS: - val = rec.fields.get(key, "") - if not val: - continue - if key in schema_fields: + td = ss.tag_map.get(rec.tag) + schema_fields = set(td.fields) if td else set() + for key, val in rec.fields.items(): + if not val or key in schema_fields: continue pair = (rec.tag, key) if pair in seen: continue seen.add(pair) - listed = " ".join(tag_def.fields) if tag_def else "" - msgs.append(f"{rec.tag}.{key} not in SCHEMA fields={listed}") + msgs.append(f"{rec.tag}.{key} fixed_tag extra cannot persist") return msgs def write_snapshot(ss: SessionStore, path: str | Path) -> int: for msg in snapshot_locator_schema_warnings(ss): emit_wrn("snapshot_schema_drop", msg) - rec_lines, hid_to_nick = _emit_record_lines(ss) - _verify_emitted_rows(ss, rec_lines, hid_to_nick) - text = _format_snapshot(ss, rec_lines) + rec_lines, hid_to_nick, emit_map = _emit_record_lines(ss) + text = _format_snapshot(ss, rec_lines, emit_map) + _verify_emitted_snapshot(ss, text, hid_to_nick, emit_map) Path(path).write_text(text, encoding="utf-8") return ss.store.row_count_non_law() @@ -304,7 +346,7 @@ def load_snapshot_text( keep_id: bool = False, ) -> SessionStore: caps = caps or Caps() - meta, map_lines, rel_lines, rec_lines = _parse_sections(text.splitlines()) + meta, map_lines, rel_lines, rec_lines = _parse_sections(split_snapshot_lines(text)) tag_map = load_persisted_map_from_lines(map_lines, caps) relations = _parse_relations(rel_lines) if not relations: diff --git a/parts/common/memnet/memnet/tag_map.py b/parts/common/memnet/memnet/tag_map.py index d0e5154..4e4b1f3 100644 --- a/parts/common/memnet/memnet/tag_map.py +++ b/parts/common/memnet/memnet/tag_map.py @@ -280,9 +280,9 @@ def parse_line( ) -> Record: caps = caps or Caps() physical = len(line.encode("utf-8")) - # Escaped wire may exceed max_line_bytes; decoded size is the documented cap. - physical_hard = max(8 * 1024 * 1024, caps.max_line_bytes) - if physical > physical_hard: + # line_bytes is the escaped/raw pipe or snapshot line (backslash and + # pipe count twice). Save verify and load share this check. + if physical > caps.max_line_bytes: raise MemNetError( "limit_exceeded", f"line_bytes|{physical}/{caps.max_line_bytes}", @@ -298,13 +298,6 @@ def parse_line( known = ",".join(tag_map.tag_names()) raise MemNetError("unknown_tag", f"{tag} not in tagMap known: {known}") values = split_payload(payload) - decoded = len(f"@{tag}: ".encode()) + sum(len(v.encode()) for v in values) - decoded += max(0, len(values) - 1) - if decoded > caps.max_line_bytes: - raise MemNetError( - "limit_exceeded", - f"line_bytes|{decoded}/{caps.max_line_bytes}", - ) fields = validate_values(tag_def, values, caps) nick = fields.get("id", "") if nick: diff --git a/parts/common/memnet/memnet/wire.py b/parts/common/memnet/memnet/wire.py index 3aac5f8..7edbc6c 100644 --- a/parts/common/memnet/memnet/wire.py +++ b/parts/common/memnet/memnet/wire.py @@ -4,6 +4,39 @@ import re +# Python str.splitlines() separators. Snapshot records split on LF only; +# these MUST be escaped so load cannot FIELD_COUNT-split a property. +_SPLITLINES_ESC: dict[str, str] = { + "\n": "\\n", + "\r": "\\r", + "\x0b": "\\v", + "\x0c": "\\f", + "\x1c": "\\x1c", + "\x1d": "\\x1d", + "\x1e": "\\x1e", + "\x85": "\\x85", + "\u2028": "\\u2028", + "\u2029": "\\u2029", +} +SPLITLINES_SEPARATORS: tuple[str, ...] = tuple(_SPLITLINES_ESC) +_HEX = frozenset("0123456789abcdefABCDEF") + + +def split_snapshot_lines(text: str) -> list[str]: + """Record split for leftover snapshots: LF only, optional CRLF trim. + + MUST NOT use str.splitlines() — that splits on CR / VT / FF / NEL / LS / PS + before unescape and yields FIELD_COUNT. + """ + if text.endswith("\n"): + text = text[:-1] + lines: list[str] = [] + for raw in text.split("\n"): + if raw.endswith("\r"): + raw = raw[:-1] + lines.append(raw) + return lines + def split_payload(payload: str) -> list[str]: if "\\" not in payload: @@ -11,9 +44,10 @@ def split_payload(payload: str) -> list[str]: fields: list[str] = [] current: list[str] = [] i = 0 - while i < len(payload): + n = len(payload) + while i < n: ch = payload[i] - if ch == "\\" and i + 1 < len(payload): + if ch == "\\" and i + 1 < n: nxt = payload[i + 1] if nxt in ("|", "\\"): current.append(nxt) @@ -27,6 +61,26 @@ def split_payload(payload: str) -> list[str]: current.append("\r") i += 2 continue + if nxt == "v": + current.append("\x0b") + i += 2 + continue + if nxt == "f": + current.append("\x0c") + i += 2 + continue + if nxt == "x" and i + 3 < n: + hx = payload[i + 2 : i + 4] + if hx[0] in _HEX and hx[1] in _HEX: + current.append(chr(int(hx, 16))) + i += 4 + continue + if nxt == "u" and i + 5 < n: + hx = payload[i + 2 : i + 6] + if all(c in _HEX for c in hx): + current.append(chr(int(hx, 16))) + i += 6 + continue if ch == "|": fields.append("".join(current)) current = [] @@ -41,13 +95,17 @@ def split_payload(payload: str) -> list[str]: def join_payload(fields: list[str]) -> str: out: list[str] = [] for field in fields: - escaped = ( - field.replace("\\", "\\\\") - .replace("|", "\\|") - .replace("\n", "\\n") - .replace("\r", "\\r") - ) - out.append(escaped) + escaped: list[str] = [] + for ch in field: + if ch == "\\": + escaped.append("\\\\") + elif ch == "|": + escaped.append("\\|") + elif ch in _SPLITLINES_ESC: + escaped.append(_SPLITLINES_ESC[ch]) + else: + escaped.append(ch) + out.append("".join(escaped)) return "|".join(out) From 2ca1bc0457737d4b7ace38618a8baab3e0a8b459 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:17:38 +0000 Subject: [PATCH 06/12] Test snapshot CR/seps round-trip, undeclared property persist, and write-time line_bytes. Co-authored-by: chouswei --- CHANGELOG.md | 2 +- docs/cap-contract.md | 6 +- docs/operations/honesty-c-wire-audit.md | 4 +- tests/cap_contract_lib.py | 2 +- tests/test_engine_roundtrip_where_acl.py | 108 ++++++++++++++++++++++- tests/test_snapshot.py | 19 ++-- tests/test_sysml_engine_bugs.py | 10 +++ 7 files changed, 134 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dfbef7..46eb6bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. - **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe and tests; no engine or cap-default change. No SemVer bump. Wire: [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md). ### Fixed -- **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes LF/CR/`|`/`\`; load decodes; save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot round-trip writes no file and RAM still drops (`snap_missing`). GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. +- **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes every `str.splitlines()` separator (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus `|`/`\`; load splits records on LF only. Undeclared RAM properties persist by widening the snapshot SCHEMA (live SCHEMA unchanged; extras on EDG/LAW or over `max_fields` fail closed). `line_bytes` is the escaped/raw line at save verify and load. Save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot round-trip writes no file and RAM still drops (`snap_missing`). GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. ### Changed - **Invent only — ClusterRoute vs SliceHandCarry (#191 / #47 cousin)** — `MemNetTwoMoves` outside `MemNetSystem` (`MN-REQ-06.9` + `MN-REQ-06.10` / `MN-VER-06-S08`). ClusterRoute = where the session lives (`MemNetLanMcpFront`; one owner; `pin_map` / `find` SHALL NOT span backends). SliceHandCarry = explicit copy into another session (`export_pin_map` or `session_save` → LAN file copy → dest import/`session_load`; `import_slice` same-serve only). Not a live hop. `import_slice(from_url)` not shipped. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/cluster-route-vs-slice-hand-carry.md`](docs/operations/cluster-route-vs-slice-hand-carry.md). diff --git a/docs/cap-contract.md b/docs/cap-contract.md index 7a72674..b00b30d 100644 --- a/docs/cap-contract.md +++ b/docs/cap-contract.md @@ -155,7 +155,11 @@ Raised at map load (`memnet/tag_map.py`). Session open fails; nothing is stored. **Value cap** is one hard cap on leftover `@TAG` `parse_line`, GQL `mutate` (CREATE / SET field values), and snapshot load. It is measured as the **UTF-8 byte length of the decoded (raw) property value**, not the escaped wire form. A product MAY raise `MEMNET_MAX_VALUE_BYTES` to `16384`. Over-cap refuses `limit_exceeded|value_bytes {n}/{max}` and does not store the row. -**Line cap** is the decoded line size (tag prefix + decoded fields + `|` separators). Snapshot emit escapes LF / CR / `|` / `\` so a value under the value cap cannot trip `line_bytes` on load because of escaping. GQL statements are not pipe lines. +**Line cap** is the **UTF-8 byte length of the escaped/raw leftover-pipe or snapshot line** (backslash and pipe count twice on the wire). Save verify and load share this check. A single field under the value cap still fits default `32768`. Many fields whose escaped form exceeds `line_bytes` refuse at save (`snapshot_unsaveable` wrapping `line_bytes`) rather than write a file load will refuse. GQL statements are not pipe lines. + +Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, FS/GS/RS, NEL, LS, PS) plus `|` and `\`. Load splits records on LF only (not `str.splitlines()`). + +**Undeclared properties.** GQL mutate may store keys that are absent from the live tag SCHEMA (GraphElement extras; Path-B locators such as `qname`). Those keys stay in RAM. Snapshot save persists them by widening the **snapshot** SCHEMA (live session SCHEMA is unchanged). After load, the restored map includes the extra columns. Extras on fixed tags `EDG` / `LAW`, or a widened SCHEMA over `max_fields`, refuse `snapshot_unsaveable` and write no file. Snapshot save verifies every emitted row can parse back to the same values. If any row cannot, save refuses `@ERR: snapshot_unsaveable|{tag} nick={nick} …` and writes no file. Expire-save in that case emits `@WRN: expire_snapshot_failed|snapshot_unsaveable`, writes no file, then RAM still drops (`session_expired|snap_missing`). Snapshots written by 0.19.18 (pipe and backslash escapes only) still load. diff --git a/docs/operations/honesty-c-wire-audit.md b/docs/operations/honesty-c-wire-audit.md index 8d01637..d8c2bc2 100644 --- a/docs/operations/honesty-c-wire-audit.md +++ b/docs/operations/honesty-c-wire-audit.md @@ -26,9 +26,9 @@ Regression: `tests/test_catalog_snap.py` (`test_cross_cut_satisfy_is_catalog_loc | CLI `query pin-map` / leftover `query warm` / MCP `pin_map` | Same composer text. | | `query find` / MCP `find` | Still seed-only; codebook miss stays empty skip (no Peak_L). CueConflict only when `total>1`. | | `export_pin_map` `conflict=` | Still true only when body contains `## CueConflict`. Peak_L miss is not `conflict=1`. | -| `write_snapshot` / `session_save` | `@WRN: snapshot_schema_drop` when RAM locator keys (`qname`, `path`, `requirementId`, `skill_id`) are absent from SCHEMA columns. SCHEMA unchanged. Path-B ingest not refused. | +| `write_snapshot` / `session_save` | **0.19.10:** `@WRN: snapshot_schema_drop` when RAM locator keys (`qname`, `path`, `requirementId`, `skill_id`) are absent from SCHEMA columns; extras vanished. **Later honesty `c` (MN-REQ-01.9):** extras persist by widening the snapshot SCHEMA; live SCHEMA unchanged; `snapshot_schema_drop` remains only for fixed-tag extras that cannot persist. | -Regression: `tests/test_peak_l.py` (`test_two_peaks_cue_miss_not_cue_conflict`); `tests/test_bounded_match_find.py` / `tests/test_honesty_c_wire.py` CueConflict; `tests/test_snapshot.py` (`test_session_save_warns_when_qname_not_in_schema`). +Regression: `tests/test_peak_l.py` (`test_two_peaks_cue_miss_not_cue_conflict`); `tests/test_bounded_match_find.py` / `tests/test_honesty_c_wire.py` CueConflict; `tests/test_snapshot.py` (`test_session_save_qname_not_in_schema_persists`). **H2 hypothesis:** warn on `write_snapshot` is enough. Foam bind used a narrow SCHEMA without `PRT.qname`; save dropped RAM `qname`; keep-id reload then missed `qname=` and Peak_L looked like CueConflict. Fix the lie on emit + warn on save. Do not silently widen SCHEMA. Nest SysML still says “two peaks → CueConflict” until a later nest pass — engine wire is Peak_L. diff --git a/tests/cap_contract_lib.py b/tests/cap_contract_lib.py index 8482a2b..6221f3e 100644 --- a/tests/cap_contract_lib.py +++ b/tests/cap_contract_lib.py @@ -513,7 +513,7 @@ def case_pipe_value_and_line_bytes() -> list[Case]: name="pipe_line_bytes", kind="hard_refuse", default="32768", - knob="MEMNET_MAX_LINE_BYTES (decoded line; leftover pipe / snapshot)", + knob="MEMNET_MAX_LINE_BYTES (escaped/raw UTF-8 of leftover pipe / snapshot line)", library_code=exc.code, library_message=exc.message, wire=format_err(exc.code, exc.message), diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py index 4c9ed73..bc2a093 100644 --- a/tests/test_engine_roundtrip_where_acl.py +++ b/tests/test_engine_roundtrip_where_acl.py @@ -13,7 +13,7 @@ from memnet.mutate_gate import MutateGate from memnet.session import get_session, open_session, snapshot_expired_session from memnet.snapshot import load_snapshot, write_snapshot -from memnet.wire import join_payload, split_payload +from memnet.wire import SPLITLINES_SEPARATORS, join_payload, split_payload runner = CliRunner() FIXTURE = Path(__file__).parent / "fixtures" / "snapshot-0.19.18.snap" @@ -35,6 +35,15 @@ def test_split_join_newlines_and_specials(): assert split_payload(wire) == [raw] +def test_split_join_all_splitlines_separators(): + raw = "ab" + "".join(SPLITLINES_SEPARATORS) + "cd\t" + wire = join_payload([raw]) + for sep in SPLITLINES_SEPARATORS: + assert sep not in wire + assert "\t" in wire + assert split_payload(wire) == [raw] + + def test_legacy_0_19_18_snapshot_loads(memnet_temp): ss = load_snapshot(FIXTURE) rec = ss.store.get("PLR_V118") @@ -85,7 +94,7 @@ def test_escaped_under_value_cap_does_not_trip_line_bytes( memnet_temp, schema_file, tmp_path: Path, monkeypatch ): monkeypatch.setenv("MEMNET_MAX_VALUE_BYTES", "64") - monkeypatch.setenv("MEMNET_MAX_LINE_BYTES", "80") + monkeypatch.setenv("MEMNET_MAX_LINE_BYTES", "200") ss = open_session(map_file=str(schema_file), caps=Caps()) blob = "n" * 20 + "\n" * 20 MutateGate(ss).apply( @@ -102,6 +111,101 @@ def test_escaped_under_value_cap_does_not_trip_line_bytes( assert loaded.store.get("PLR_ESC").fields["identity"] == blob +def test_snapshot_roundtrip_all_splitlines_separators( + memnet_temp, schema_file, tmp_path: Path +): + ss = open_session(map_file=str(schema_file)) + blob = "ab\rcd" + "".join(SPLITLINES_SEPARATORS) + "\tab\r" + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR_CR', identity: 'x', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + rec = ss.store.get("PLR_CR") + assert rec is not None + rec.fields["identity"] = blob + path = tmp_path / "seps.snap" + write_snapshot(ss, path) + text = path.read_text(encoding="utf-8") + rec_line = next(ln for ln in text.split("\n") if ln.startswith("@PLR:")) + for sep in SPLITLINES_SEPARATORS: + assert sep not in rec_line + loaded = load_snapshot(path) + got = loaded.store.get("PLR_CR") + assert got is not None + assert got.fields["identity"] == blob + + +def test_snapshot_persists_undeclared_properties(memnet_temp, tmp_path: Path): + ss = open_session( + map_lines=["SCHEMA CST ; fields=id name role"], + ) + MutateGate(ss).apply( + ["CREATE (:CST {id: 'N_X', name: 'keep', role: 'r', extra_k: 'extra_v'})"], + mode="add", + ) + rec = ss.store.get("N_X") + assert rec is not None + assert rec.fields.get("extra_k") == "extra_v" + path = tmp_path / "extra.snap" + write_snapshot(ss, path) + text = path.read_text(encoding="utf-8") + assert "SCHEMA CST ; fields=id name role extra_k" in text + assert "extra_v" in text + loaded = load_snapshot(path) + got = loaded.store.get("N_X") + assert got is not None + assert got.fields.get("extra_k") == "extra_v" + assert "extra_k" in loaded.tag_map.get("CST").fields + + +def test_snapshot_refuses_extras_over_max_fields( + memnet_temp, tmp_path: Path, monkeypatch +): + monkeypatch.setenv("MEMNET_MAX_FIELDS", "4") + ss = open_session( + map_lines=["SCHEMA CST ; fields=id name role"], + caps=Caps(), + ) + MutateGate(ss).apply( + [ + "CREATE (:CST {id: 'N_F', name: 'a', role: 'b', " + "extra1: 'x', extra2: 'y'})" + ], + mode="add", + ) + path = tmp_path / "fields.snap" + with pytest.raises(MemNetError) as ei: + write_snapshot(ss, path) + assert ei.value.code == "snapshot_unsaveable" + assert "fields|" in ei.value.message + assert not path.exists() + + +def test_snapshot_refuses_escaped_line_over_line_bytes( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + monkeypatch.setenv("MEMNET_MAX_LINE_BYTES", "80") + ss = open_session(map_file=str(schema_file), caps=Caps()) + blob = "|" * 40 + MutateGate(ss).apply( + [ + "CREATE (:PLR {id: 'PLR_LN', identity: '" + + _gql_escape(blob) + + "', wealth: 1, cashflow: 0, monopoly: 0, reputation: 0, inventory: 'bag'})" + ], + mode="add", + ) + path = tmp_path / "lineb.snap" + with pytest.raises(MemNetError) as ei: + write_snapshot(ss, path) + assert ei.value.code == "snapshot_unsaveable" + assert "line_bytes" in ei.value.message + assert not path.exists() + + def test_where_contains_filters_and_false_set_is_noop(memnet_temp, schema_file): ss = open_session(map_file=str(schema_file)) MutateGate(ss).apply([_PLR], mode="add") diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index dcb8211..a159f27 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -166,10 +166,9 @@ def test_mission_empty_nick_infile_save_load(memnet_temp, tmp_path: Path): assert "invalid_relation" not in load.stderr -def test_session_save_warns_when_qname_not_in_schema(memnet_temp, tmp_path: Path): - """H2: RAM qname is dropped on emit_record if SCHEMA omits it — warn, do not rewrite SCHEMA.""" +def test_session_save_qname_not_in_schema_persists(memnet_temp, tmp_path: Path): + """Undeclared RAM extras persist by widening snapshot SCHEMA (MN-REQ-01.9).""" del memnet_temp - from memnet.output import reset_warn_budget from memnet.snapshot import snapshot_locator_schema_warnings narrow = [ @@ -185,11 +184,8 @@ def test_session_save_warns_when_qname_not_in_schema(memnet_temp, tmp_path: Path ) prt = ss.store.list_records("PRT")[0] assert prt.fields.get("qname") == "Pkg::Valve" - warns = snapshot_locator_schema_warnings(ss) - assert any("PRT.qname" in w for w in warns) - assert not any("PRT.path" in w for w in warns) + assert snapshot_locator_schema_warnings(ss) == [] - reset_warn_budget() snap_path = tmp_path / "narrow.snap" save = runner.invoke( app, @@ -197,10 +193,13 @@ def test_session_save_warns_when_qname_not_in_schema(memnet_temp, tmp_path: Path ) assert save.exit_code == 0, save.stderr mixed = save.stderr + save.stdout - assert "snapshot_schema_drop" in mixed - assert "PRT.qname" in mixed + assert "snapshot_schema_drop" not in mixed text = snap_path.read_text(encoding="utf-8") - assert "Pkg::Valve" not in text + assert "Pkg::Valve" in text + assert "qname" in text + loaded = load_snapshot(snap_path) + got = next(r for r in loaded.store.list_records("PRT") if r.fields.get("name") == "Valve") + assert got.fields.get("qname") == "Pkg::Valve" wide = open_session(map_lines=_MISSION_MAP) _mission_graph(wide) diff --git a/tests/test_sysml_engine_bugs.py b/tests/test_sysml_engine_bugs.py index e42aaea..3f65092 100644 --- a/tests/test_sysml_engine_bugs.py +++ b/tests/test_sysml_engine_bugs.py @@ -18,6 +18,9 @@ def test_requirements_ids_present(): assert "snapshot_unsaveable" in text assert "unsupported_predicate" in text assert "MEMNET_MAX_VALUE_BYTES" in text + assert "str.splitlines()" in text + assert "widening the" in text + assert "escaped/raw leftover-pipe" in text assert "snapshotLosslessRoundTripReq" in text assert "sessionLifecycleAclWhoReq" in text assert "honourWherePredicateReq" in text @@ -32,8 +35,15 @@ def test_deploy_and_verify_trail_model(): assert 'envMaxValueBytes : String = "MEMNET_MAX_VALUE_BYTES"' in deploy assert "sessionSaveLoadCloseAclWho" in deploy assert "acceptsCaller" in deploy + assert "persistUndeclaredProperties" in deploy + assert "splitlinesSeparatorsEscaped" in deploy + assert "recordSplitLfOnly" in deploy + assert "lineBytesOnEmittedLine" in deploy ver = VERIFY.read_text(encoding="utf-8") assert "MN-VER-01-S04" in ver assert "MN-VER-01-S05" in ver assert "MN-VER-03-S01" in ver assert "MN-VER-05-S01" in ver + assert "persistUndeclaredProperties" in ver + assert "splitlinesSeparatorsEscaped" in ver + assert "lineBytesOnEmittedLine" in ver From 1b4d918e9ef6ac1fc856b0e1a6fea3803a371f48 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:21:49 +0000 Subject: [PATCH 07/12] Format E18 snapshot tests and shorten cap-contract line_bytes knob. Co-authored-by: chouswei --- tests/cap_contract_lib.py | 2 +- tests/test_engine_roundtrip_where_acl.py | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/cap_contract_lib.py b/tests/cap_contract_lib.py index 6221f3e..4b0785e 100644 --- a/tests/cap_contract_lib.py +++ b/tests/cap_contract_lib.py @@ -513,7 +513,7 @@ def case_pipe_value_and_line_bytes() -> list[Case]: name="pipe_line_bytes", kind="hard_refuse", default="32768", - knob="MEMNET_MAX_LINE_BYTES (escaped/raw UTF-8 of leftover pipe / snapshot line)", + knob="MEMNET_MAX_LINE_BYTES (escaped/raw leftover pipe / snapshot line)", library_code=exc.code, library_message=exc.message, wire=format_err(exc.code, exc.message), diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py index bc2a093..f588756 100644 --- a/tests/test_engine_roundtrip_where_acl.py +++ b/tests/test_engine_roundtrip_where_acl.py @@ -111,9 +111,7 @@ def test_escaped_under_value_cap_does_not_trip_line_bytes( assert loaded.store.get("PLR_ESC").fields["identity"] == blob -def test_snapshot_roundtrip_all_splitlines_separators( - memnet_temp, schema_file, tmp_path: Path -): +def test_snapshot_roundtrip_all_splitlines_separators(memnet_temp, schema_file, tmp_path: Path): ss = open_session(map_file=str(schema_file)) blob = "ab\rcd" + "".join(SPLITLINES_SEPARATORS) + "\tab\r" MutateGate(ss).apply( @@ -161,19 +159,14 @@ def test_snapshot_persists_undeclared_properties(memnet_temp, tmp_path: Path): assert "extra_k" in loaded.tag_map.get("CST").fields -def test_snapshot_refuses_extras_over_max_fields( - memnet_temp, tmp_path: Path, monkeypatch -): +def test_snapshot_refuses_extras_over_max_fields(memnet_temp, tmp_path: Path, monkeypatch): monkeypatch.setenv("MEMNET_MAX_FIELDS", "4") ss = open_session( map_lines=["SCHEMA CST ; fields=id name role"], caps=Caps(), ) MutateGate(ss).apply( - [ - "CREATE (:CST {id: 'N_F', name: 'a', role: 'b', " - "extra1: 'x', extra2: 'y'})" - ], + ["CREATE (:CST {id: 'N_F', name: 'a', role: 'b', extra1: 'x', extra2: 'y'})"], mode="add", ) path = tmp_path / "fields.snap" From bb26246776cfd823c6ff0c7e40fd3722d65cc9fd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:36:48 +0000 Subject: [PATCH 08/12] Flip #201 doc-gate probe and tests onto lossless snapshot, honoured WHERE, and ACL who. Co-authored-by: chouswei --- CHANGELOG.md | 2 +- docs/operations/one-session-per-document.md | 22 +-- scripts/probe_doc_gate_readiness.py | 160 ++++++++++---------- tests/doc_gate_lib.py | 45 +++--- tests/test_doc_gate_readiness.py | 102 +++++++------ 5 files changed, 182 insertions(+), 149 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46eb6bc..fe3eb84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ## [Unreleased] ### Added -- **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe and tests; no engine or cap-default change. No SemVer bump. Wire: [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md). +- **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe, tests, and [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md) now match the engine fix (lossless snapshot, honoured WHERE, ACL who on save/load/close). No SemVer bump. ### Fixed - **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes every `str.splitlines()` separator (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus `|`/`\`; load splits records on LF only. Undeclared RAM properties persist by widening the snapshot SCHEMA (live SCHEMA unchanged; extras on EDG/LAW or over `max_fields` fail closed). `line_bytes` is the escaped/raw line at save verify and load. Save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot round-trip writes no file and RAM still drops (`snap_missing`). GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. diff --git a/docs/operations/one-session-per-document.md b/docs/operations/one-session-per-document.md index 08b9caf..1eb071f 100644 --- a/docs/operations/one-session-per-document.md +++ b/docs/operations/one-session-per-document.md @@ -21,7 +21,7 @@ Settings this gate uses: | Concurrent sessions | 1024 (`MEMNET_MAX_SESSIONS`) | | Document size | about 1 800 part nodes plus a few opaque `USR` text nodes | -Ingest caps (`max_nodes=2000` / `max_edges=2000`) are Path-B ingest, not this mutate path. Session row cap remains 5000 non-LAW rows (**nodes plus edges**). `session load` of a snapshot is **not** bound by ingest budget; it walks leftover `parse_line` + `MemStore.upsert` (`MEMNET_MAX_ROWS`, max sessions, leftover value/line/newline/FIELD_COUNT). +Ingest caps (`max_nodes=2000` / `max_edges=2000`) are Path-B ingest, not this mutate path. Session row cap remains 5000 non-LAW rows (**nodes plus edges**). `session load` of a snapshot is **not** bound by ingest budget; it walks leftover `parse_line` + `MemStore.upsert` (`MEMNET_MAX_ROWS`, max sessions, leftover value/line/`FIELD_COUNT` on a malformed pipe line). ## Request envelope (direct serve) @@ -45,13 +45,13 @@ Client helper: `memnet.serve.send_command(args, stdin=…, host=…, port=…)`. `session save --file` writes `# memnet-snapshot-v1` via `Path.write_text` (overwrite). MemNet does **not** make that file write-once (no `O_EXCL`, no `chmod`, no immutable flag). The caller or the filesystem can. -Opaque text must stay on one snapshot line. Newlines inside a property survive GQL mutate in RAM, but leftover `@TAG` emit does not escape them, so `session load` raises `@ERR: FIELD_COUNT`. Leftover emit **does** escape `\` and `|` (`join_payload`). A 16 KiB string survives CREATE / SET / `pin_map` in RAM (GQL mutate does not enforce pipe `value_bytes` / `line_bytes` — cap-contract bug 4) but snapshot load of a 16 KiB field refuses `@ERR: limit_exceeded|value_bytes 16384/4096`. Unicode, `|`, and quotes on a **short** single line do round-trip. +Anything mutate accepts must save and reload as the same property values. Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, FS/GS/RS, NEL, LS, PS) plus `\` and `|`; load splits records on LF only. A 16 KiB string is refused at GQL CREATE/SET with `@ERR: limit_exceeded|value_bytes 16384/4096` (one decoded cap with leftover pipe and snapshot load). Unicode, `|`, quotes, CR, and newlines on a value under the cap round-trip. Undeclared RAM keys persist by widening the snapshot SCHEMA; extras over `max_fields` refuse `snapshot_unsaveable`. Expire: with save-on-expire and a dir, TTL drop writes `{dir}/{sid}.snap` (do not log the name). Next use: `@ERR: session_expired|snap_available`. Restore: `session load --session ` (no `--file`). ## ACL -CapsPolicy is off until grant/enable. Who and WorkerWriteScope apply to `pin_map`, `mutate`, and `export pin-map` when ACL is on. `session save` / `load` / `close` do not take `--caller` and do not who-check. Bind is skipped on `memnet serve` (`MEMNET_SERVE_INTERNAL=1`). +CapsPolicy is off until grant/enable. Who and WorkerWriteScope apply to `pin_map`, `mutate`, and `export pin-map` when ACL is on. `session save` / `load` into an ACL'd session / `close` accept `--caller` / `MEMNET_CALLER` and who-check with the same codes. Bind is skipped on `memnet serve` (`MEMNET_SERVE_INTERNAL=1`). ## Memory figures @@ -74,10 +74,10 @@ python scripts/probe_doc_gate_readiness.py --out /opt/cursor/artifacts/doc-gate- |------|------------------------| | E11 | `session load` of a 3000-node snapshot (mutate batches ≤1000 lines, then save/close/load) is **not** `ingest_budget`. Bound by `MEMNET_MAX_ROWS` (5000) at upsert. Neither batched load nor an ingest exemption is needed at 3000. | | E12 | **yes** (revised, fulldoc). Counts **nodes plus edges** on write and `session_load`. 3000 nodes then 2000 edges fill default 5000; next edge `@ERR: limit_exceeded\|rows 5001/5000`. Pi 10000 holds 7500 (`rows=7500` `edges=4500`) and `session load` of that snapshot is **not** `ingest_budget` (loaded 7500). Same 7500 snapshot on 5000: `@ERR: limit_exceeded\|rows 5001/5000`. `pin_map` read is **not** the session cap: hub `M=50` → `## Truncation truncated=true M=50 omitted=2956 reason=max_rows`; hub `M=4000` → `@ERR: response_too_large\|response 9491260 bytes exceeds cap 4194304` (4 MiB serve frame). | -| E13 | 16 KiB strings with LaTeX / quotes / newline / `\|` / CJK survive GQL CREATE/SET/`pin_map` in RAM. Snapshot save/load does **not** survive byte-for-byte (`FIELD_COUNT` on newlines; `value_bytes 16384/4096` otherwise). Pipe leftover: value 4096, line 32768; GQL mutate skips those (bug 4). Escapes: `\\ \' \" \n \r \t` only. | -| E14 | List literals store as JSON strings and emit as GQL lists. `'k' IN p.citeKeys` is **not** a product filter (`MATCH (p:USR) WHERE … SET` ignores WHERE and raises `cue_conflict` when \|Q\|>1). Locators are `KEY=VAL` equality on the JSON string; leftover `read list --where` can glob that string. | -| E17 | **note.** `WHERE n.value CONTAINS` is not a product filter. RETURN → `product_gate`; SET drops WHERE (`cue_conflict` at \|Q\|=1500; unique MATCH still SET on a miss). `STARTS WITH` / `ENDS WITH` / `=~` same. Working: `find`/`pin_map --keyword` (casefold). See paragraph below. | -| E18 | Snapshot `value_bytes` 4096 is the **decoded** field after `split_payload` (`>` not `>=`); `join_payload` expansion of `\\` / `\|` is not the cap. `line_bytes` 32768 is the raw snapshot line. SCHEMA `max_fields=32`. Tab round-trips; CR/LF do not. Live table below. | +| E13 | 16 KiB strings with LaTeX / quotes / newline / `\|` / CJK are refused at GQL CREATE/SET (`value_bytes 16384/4096`). Pipe leftover, GQL mutate, and snapshot load share decoded value 4096. `line_bytes` 32768 is leftover-pipe / snapshot escaped line. GQL string escapes: `\\ \' \" \n \r \t`. | +| E14 | List literals store as JSON strings and emit as GQL lists. `'k' IN p.citeKeys` honours membership on MATCH…SET. Locators are `KEY=VAL` equality on the JSON string; leftover `read list --where` can glob that string. | +| E17 | `WHERE n.value CONTAINS` is honoured on SET/DELETE. RETURN → `product_gate`. Unique MATCH miss → `not_found` (nothing SET). SET of \|Q\|>1 after WHERE is still `cue_conflict` (unique-SET law). `STARTS WITH` / `ENDS WITH` / `=~` same. Keyword: `find`/`pin_map --keyword` (casefold). See paragraph below. | +| E18 | Snapshot `value_bytes` 4096 is the **decoded** field after `split_payload` (`>` not `>=`); `join_payload` expansion of splitlines separators / `\\` / `\|` is not the value cap. `line_bytes` 32768 is the escaped/raw snapshot line. SCHEMA `max_fields=32`. Tab, CR, and LF round-trip. Undeclared extras persist unless they exceed `max_fields`. Live table below. | Second RSS fixture: 3000 nodes, no edges, 1500 of them with 2/3/4 KiB text (about 4.4 MiB of UTF-8 payload, not 1 MiB). Measure process RSS the same way as the 1800-part fixture. Short fat churn is on (`--fat-churn`, default 8); 110 cycles of this fixture is not the default. @@ -92,7 +92,7 @@ E16 (on the 10000 fulldoc session, this VM `Intel(R) Xeon(R) Processor` 4-core K **note:** MemNet has **no** native delete-refused-while-referenced check. `DETACH DELETE` of a node with inbound edges exits 0 and leaves dangling edges. The gate must refuse from the reverse lookup (`## Truncation truncated=true M=400 omitted=2604 reason=max_rows` on the hub). Documented `MATCH ()-[r {id}]-() DELETE r` is lowered as a node DROP with an empty id and refuses `@ERR: not_found|DELETE matched no element` (not a referenced-delete check). The probe's working edge DROP is `MATCH (n WHERE true)-[r {id}]->() DELETE r`. -E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_payload` (`tag_map.validate_values`: `len(val.encode("utf-8")) > caps.max_value_bytes`, so 4096 passes). `join_payload` expansion of `\\` / `|` is not the cap (4000 `\\` or `|` emit 8000 escaped bytes, snap line ~8021, still loads). `parse_line` measures raw line vs `line_bytes` 32768 first. SCHEMA register vs `max_fields=32`. `emit_record` writes SCHEMA columns only. Loopback CREATE → save → load into a fresh session → `pin_map` cue. Binary search skipped (4000 exact for `\\` and `|`). Proof: `/opt/cursor/artifacts/doc-gate-readiness-e18.log`. +E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_payload` (`check_value_bytes`: decoded length `>` cap, so 4096 passes). `join_payload` expansion of splitlines separators / `\\` / `|` is not the value cap (4000 `\\` or `|` emit 8000 escaped bytes, snap line ~8021, still loads). `parse_line` measures the escaped/raw line vs `line_bytes` 32768 first; save verify uses the same check. SCHEMA register vs `max_fields=32`. Snapshot save widens SCHEMA for undeclared RAM keys; leftover `emit_record` still writes SCHEMA columns. Loopback CREATE → save → load into a fresh session → `pin_map` cue. Binary search skipped (4000 exact for `\\` and `|`). | Case | Wire shape | Save / load | Exact? | |------|------------|-------------|--------| @@ -103,10 +103,10 @@ E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_ | E18a 4000 CJK | 1333 × U+6D4B (`测`, 3-byte UTF-8) + 1 ASCII X; `USR_e4kcj` | 0 / 0 | **yes** (1334 chars, utf8 4000, snap line 4021) | | E18a 4096 (a–e) | same shapes; CJK is 1365 × `测` + 1 X | 0 / 0 all five | **yes** (`\\`/`\|` snap line 8215; quotes/CJK 4119) | | E18b tab mid/end | `CREATE (:USR {id: 'USR_tabm', … value: 'ab\\tcd'})` / `'ab\\t'` | 0 / 0 | **yes** (byte-exact) | -| E18b CR mid/end | `… value: 'ab\\rcd'` / `'ab\\r'` | save 0 / load 1 | **no** — `@ERR: FIELD_COUNT\|Expected 4 fields for USR got 3` (same as newline: `str.splitlines` splits on CR before `newline_in_value`) | +| E18b CR mid/end | `… value: 'ab\\rcd'` / `'ab\\r'` | save 0 / load 0 | **yes** (byte-exact; LF-only record split) | | E18c SCHEMA 64/128 | `SCHEMA WIDE ; fields=id p000 …` (64 / 128 names) | open 1 | `@ERR: limit_exceeded\|fields 64/32` and `128/32` | | E18c SCHEMA 32 | `SCHEMA PRT ; fields=id p00 … p30` | save 0 / load 0 | yes | -| E18c 64/128 extras on USR | CREATE 60 / 124 keys beyond 4-field SCHEMA | save 0 / load 0 | RAM extras yes; load drops them (`emit_record` SCHEMA columns only) | +| E18c 64/128 extras on USR | CREATE 60 / 124 keys beyond 4-field SCHEMA | save 1 | RAM extras yes; save refuses `snapshot_unsaveable` (`fields\|n/32`). Extras that fit persist by widening snapshot SCHEMA. | | E18c 8 × 4000 ASCII | `CREATE (:FAT {id: 'FAT_8x4000', p000: <4000 A>, … p007: <4000 A>})` | 0 / 0 | **yes** (snap line 32024 < 32768; all eight fields exact) | -E17: **note.** GQL `WHERE n.value CONTAINS '…'` is **not** a product substring filter. `MATCH … WHERE … RETURN n` → `@ERR: product_gate|agent surface forbids RETURN …`. `MATCH … WHERE … SET` GraphGlot-parses (single or double quotes; GQL escapes `\\ \' \" \n \r \t`; CJK and `$` unescaped) but lowering drops WHERE at SET: `|Q|=1500` → `@ERR: cue_conflict|SET Q =1500; SHALL NOT pick one root or absorb`; a unique MATCH still SET when CONTAINS would miss. Inline `MATCH (n WHERE n.value CONTAINS '…')` → `@ERR: parse_error|unsupported MATCH shape`. Bare WHERE without SET/RETURN → `@ERR: parse_error|unsupported MATCH continuation`. `STARTS WITH` / `ENDS WITH` / `=~` are the same ignored-WHERE SET path. Working substring: `query find --keyword` / `pin_map --keyword` (casefold across all fields, hard `--limit` / `--max-rows`). leftover `read list --where value=*测例*` works on a small graph; on this fulldoc it is `@ERR: response_too_large|response 4659616 bytes exceeds cap 4194304`. Substitute latency (n=200, `find --limit 50`, warm 10000-row session, this VM): common `测例` (1500 USR hits, 50 returned) p50 **67.282** / p95 **84.020** / max **94.672** ms; rare `Part 1500` (1 hit) p50 **51.183** / p95 **69.090** / max **85.424** ms. +E17: **yes.** GQL `WHERE n.value CONTAINS '…'` is honoured on SET/DELETE. `MATCH … WHERE … RETURN n` → `@ERR: product_gate|agent surface forbids RETURN …`. Unique MATCH miss → `@ERR: not_found` and nothing SET. SET of `|Q|>1` after WHERE is still `@ERR: cue_conflict|SET Q =…` (unique-SET law, not a dropped WHERE). Inline `MATCH (n WHERE n.value CONTAINS '…')` → `@ERR: parse_error|unsupported MATCH shape`. Bare WHERE without SET/RETURN → `@ERR: parse_error|unsupported MATCH continuation`. `STARTS WITH` / `ENDS WITH` / `=~` are honoured the same way. Keyword substring: `query find --keyword` / `pin_map --keyword` (casefold across all fields, hard `--limit` / `--max-rows`). leftover `read list --where value=*测例*` works on a small graph; on this fulldoc it is `@ERR: response_too_large|response 4659616 bytes exceeds cap 4194304`. Substitute latency (n=200, `find --limit 50`, warm 10000-row session, this VM): common `测例` (1500 USR hits, 50 returned) p50 **67.282** / p95 **84.020** / max **94.672** ms; rare `Part 1500` (1 hit) p50 **51.183** / p95 **69.090** / max **85.424** ms. diff --git a/scripts/probe_doc_gate_readiness.py b/scripts/probe_doc_gate_readiness.py index af21426..ee18b91 100644 --- a/scripts/probe_doc_gate_readiness.py +++ b/scripts/probe_doc_gate_readiness.py @@ -337,21 +337,20 @@ def item3_roundtrip(svc: ServeProc, tmp: Path, *, n_parts: int) -> ItemResult: load_nl = svc.load_file(snap_nl) wires.extend(err_lines(extra.stderr) + err_lines(save_nl.stderr) + err_lines(load_nl.stderr)) multiline_field_count = any("FIELD_COUNT" in e for e in err_lines(load_nl.stderr)) - if load_nl.exit_code == 0: - gaps.append("multi-line text unexpectedly loaded") - elif not multiline_field_count: - gaps.append("multi-line load failed but not FIELD_COUNT") + multiline_ok = load_nl.exit_code == 0 and extra.exit_code == 0 and save_nl.exit_code == 0 + if not multiline_ok: + gaps.append("multi-line text failed to save/load") + if multiline_field_count: + gaps.append("multi-line load still FIELD_COUNT") else: notes.append( - "Raw newlines in a property survive mutate in RAM; leftover snapshot " - "emit does not escape them, so load raises FIELD_COUNT." + "Newlines in a property escape on snapshot emit; load splits LF-only " + "and round-trips the value." ) if not exact: gaps.append("canonical dump differed after save/load (single-line text)") - verdict = "note" if exact and multiline_field_count else ("yes" if exact else "no") - if exact and multiline_field_count: - verdict = "note" + verdict = "yes" if exact and multiline_ok else "no" return ItemResult( item="3 Snapshot round-trip", verdict=verdict, @@ -363,6 +362,7 @@ def item3_roundtrip(svc: ServeProc, tmp: Path, *, n_parts: int) -> ItemResult: "exact_canonical_single_line": exact, "multiline_load_exit": load_nl.exit_code, "multiline_field_count": multiline_field_count, + "multiline_ok": multiline_ok, "src_bytes": len(src_text), "write_once": wo, }, @@ -443,12 +443,14 @@ def item5_acl(svc: ServeProc) -> ItemResult: bind = svc.acl_bind(sid, "mission-a", "lease-a") checks: dict[str, dict[str, Any]] = {} - lifecycle = { + lifecycle_must_acl = { "save missing caller", + "close missing caller", + } + lifecycle_must_ok = { "save with caller", "load with caller", - "close missing caller", - "bind mutate without mission/lease", + "close with caller", } def rec(name: str, reply: ServeReply, *, expect_acl: bool) -> None: @@ -459,7 +461,7 @@ def rec(name: str, reply: ServeReply, *, expect_acl: bool) -> None: for ln in (reply.stderr or "").splitlines() ) acl_hit = any(e.startswith("@ERR: acl_") for e in errs) - skipped = name in lifecycle and not acl_hit + skipped = name in lifecycle_must_acl and not acl_hit checks[name] = { "exit": reply.exit_code, "acl_hit": acl_hit, @@ -530,12 +532,13 @@ def rec(name: str, reply: ServeReply, *, expect_acl: bool) -> None: svc.save(sid, svc.snap_dir / "acl-save2.snap", caller="owner"), expect_acl=True, ) - rec( - "load with caller", - svc.load_file(svc.snap_dir / "acl-save.snap", caller="owner"), - expect_acl=True, - ) + load_acl = svc.load_file(svc.snap_dir / "acl-save2.snap", caller="owner") + rec("load with caller", load_acl, expect_acl=True) + loaded_sid = _sid_from(load_acl.stdout) + if loaded_sid: + svc.close(loaded_sid) rec("close missing caller", svc.close(sid), expect_acl=True) + rec("close with caller", svc.close(sid, caller="owner"), expect_acl=True) # Bind skip: reopen, grant+bind, mutate without mission/lease through serve. sid2 = svc.open_session() @@ -555,9 +558,14 @@ def rec(name: str, reply: ServeReply, *, expect_acl: bool) -> None: skipped = [k for k, v in checks.items() if v.get("skipped")] checked = [k for k, v in checks.items() if v.get("acl_hit")] gaps = [f"ACL not checked: {k}" for k in skipped] + for name in lifecycle_must_ok: + row = checks.get(name) or {} + if row.get("exit") != 0: + gaps.append(f"ACL lifecycle should succeed: {name}") notes = [ f"grant exit={grant.exit_code} bind exit={bind.exit_code}", "Bind is skipped on serve (MEMNET_SERVE_INTERNAL=1) — confirmed below.", + "session save / close who-check when ACL is enabled (`--caller`).", f"bind_skipped={bind_skipped}", ] wires = [] @@ -567,9 +575,11 @@ def rec(name: str, reply: ServeReply, *, expect_acl: bool) -> None: who_ok = any("acl_who" in e for row in checks.values() for e in row["errs"]) denied_ok = any("acl_denied" in e for row in checks.values() for e in row["errs"]) scope_ok = any("acl_scope" in e for row in checks.values() for e in row["errs"]) - verdict = "note" if skipped else "yes" + lifecycle_ok = not skipped and all( + (checks.get(k) or {}).get("exit") == 0 for k in lifecycle_must_ok + ) + verdict = "yes" if (who_ok and denied_ok and lifecycle_ok) else "no" if not (who_ok and denied_ok): - verdict = "no" gaps.append("missing acl_who and/or acl_denied on pin_map/mutate") return ItemResult( item="5 Per-session ACL over serve", @@ -958,12 +968,16 @@ def item_e13_strings(svc: ServeProc, tmp: Path) -> ItemResult: props = shaped_node_props(pin.stdout) or {} got = props.get("value") ram_ok = create.exit_code == 0 and setted.exit_code == 0 and got == blob + over_cap = any("value_bytes" in e for e in err_lines(create.stderr)) cases["pin_map_roundtrip"] = ram_ok cases["pin_map_exit"] = pin.exit_code cases["pin_map_value_bytes"] = len(got.encode("utf-8")) if isinstance(got, str) else None - if not ram_ok: - gaps.append("16 KiB special blob did not round-trip CREATE/SET/pin_map") + cases["create_value_bytes"] = over_cap + if ram_ok: + gaps.append("16 KiB special blob was accepted by GQL mutate (value_bytes should refuse)") wires.extend(err_lines(pin.stderr)) + elif not over_cap: + gaps.append("16 KiB CREATE refused but not value_bytes") bad_esc = svc.mutate( sid, @@ -1020,28 +1034,20 @@ def item_e13_strings(svc: ServeProc, tmp: Path) -> ItemResult: } wires.extend(err_lines(load3.stderr)) - snap_ok = load.exit_code == 0 and load2.exit_code == 0 and load3.exit_code == 0 - if snap_ok: - gaps.append("16 KiB snapshot load unexpectedly succeeded for all variants") notes = [ "GQL string literals: single or double quotes; escapes are \\\\ \\' \\\" \\n \\r \\t only. " - "Unknown escape -> parse_error (GraphGlot/ParseError). gql mutate does not enforce " - "MEMNET_MAX_VALUE_BYTES=4096 or MEMNET_MAX_LINE_BYTES=32768 (cap-contract bug 4).", - "Leftover parse_line: value_bytes 4096 -> @ERR: limit_exceeded|value_bytes {n}/{max} " - "(inner pipe becomes space); line_bytes 32768 -> limit_exceeded|line_bytes; " - "newline_in_value; FIELD_COUNT. Mutate stdin: batch_lines 1000. Serve frame 4 MiB.", + "Unknown escape -> parse_error (GraphGlot/ParseError). GQL mutate, leftover pipe, " + "and snapshot load share decoded MEMNET_MAX_VALUE_BYTES=4096 " + "(`limit_exceeded|value_bytes n/max`). line_bytes 32768 is leftover-pipe / " + "snapshot escaped line, not a GQL statement cap.", "ISO INSERT is not the mutate spelling (CREATE is).", ] - if ram_ok and not snap_ok: - verdict = "note" + if not ram_ok and over_cap: + verdict = "yes" notes.append( - "16 KiB survives CREATE/SET/pin_map in RAM (bug 4). Snapshot save/load does not " - "round-trip byte-for-byte. Newlines split leftover pipe lines (FIELD_COUNT). " - "Leftover emit escapes | as \\| so a 16 KiB value with pipes still hits " - "value_bytes 16384/4096, same as a plain 16 KiB value." + "16 KiB CREATE/SET refuse at write time with value_bytes 16384/4096. " + "Nothing is stored; snapshot save of that session does not write the blob." ) - elif ram_ok and snap_ok: - verdict = "yes" else: verdict = "no" return ItemResult( @@ -1136,24 +1142,22 @@ def item_e14_lists(svc: ServeProc) -> ItemResult: notes = [ "GQL parser accepts [a, b] lists; _value_to_store json.dumps them into a string field. " "pin_map re-parses JSON-looking [ ] on emit.", - "pin_map / find locators are KEY=VAL exact equality (no IN membership). leftover " + "pin_map / find locators are KEY=VAL exact equality. leftover " "read list --where is field=value with * ? glob on the JSON string.", - "MATCH…WHERE is not a product mutate form; leftover lowering has no IN operator.", + "MATCH WHERE 'k' IN p.citeKeys SET honours membership; a miss is not SET.", ] if membership_works: verdict = "yes" - notes.append("WHERE 'k' IN p.citeKeys unexpectedly filtered (product IN).") elif store_ok and not membership_works: - verdict = "note" + verdict = "no" if where_ignored: notes.append( "WHERE 'k' IN p.citeKeys SET applied to every matched USR (WHERE ignored)." ) gaps.append("WHERE IN is ignored on leftover MATCH…SET (not membership filter)") elif in_mut.exit_code != 0: - notes.append( - "WHERE IN mutate refused (see wire). Lists store; membership filter does not." - ) + notes.append("WHERE IN mutate refused (see wire).") + gaps.append("WHERE IN did not filter") if not loc_hit: notes.append("locator equality on the JSON string is the only pin_map list lookup.") else: @@ -1828,46 +1832,42 @@ def _count_shaped(stdout: str) -> int: common_sum = latency_summary(common_samples) rare_sum = latency_summary(rare_samples) - contains_filters = False + contains_filters = not where_ignored ret_gate = any( "product_gate" in e and "RETURN" in e for e in form_results["contains_return"]["errs"] ) - set_conflict = any("cue_conflict" in e for e in form_results["contains_set_squote_cjk"]["errs"]) starts_same = any("cue_conflict" in e for e in form_results["starts_with"]["errs"]) regex_same = any("cue_conflict" in e for e in form_results["regex"]["errs"]) keyword_ok = common_find.exit_code == 0 and common_n > 0 and rare_find.exit_code == 0 notes = [ f"cpu_model={cpu}", "GQL MATCH … WHERE n.value CONTAINS '…' RETURN n is refused " - "(product_gate forbids RETURN). MATCH … WHERE … SET parses, but " - "_split_match_body cuts at SET and dropping WHERE; |Q|>1 is cue_conflict; " - "a unique MATCH still SET when CONTAINS would miss.", + "(product_gate forbids RETURN). MATCH … WHERE … SET honours the " + "predicate; a unique MATCH miss is not_found and does not SET. " + "SET of |Q|>1 after WHERE is still cue_conflict (unique-SET law), " + "not a dropped WHERE.", "Needle escaping is GQL string rules only (\\\\ \\' \\\" \\n \\r \\t). " "CJK and $ need no escape. Both '…' and \"…\" parse for the CONTAINS " - "operand. A single quote inside a single-quoted needle is \\'; a double " - "quote sits in a single-quoted needle as '\"' or a single quote in " - 'double quotes as "\'". This does not make CONTAINS a filter.', - "Working substring: query find --keyword / pin_map --keyword " + "operand.", + "Keyword substring: query find --keyword / pin_map --keyword " "(casefold haystack; explicit --limit / --max-rows). leftover " "read list --where field=*glob* works on a small graph; listing " - "1500 fat USR values can hit the 4 MiB serve frame. STARTS WITH / " - "ENDS WITH / =~ are the same ignored-WHERE SET path, not filters.", - "Latency below is find --keyword on the warm fulldoc (not CONTAINS).", + "1500 fat USR values can hit the 4 MiB serve frame.", + "Latency below is find --keyword on the warm fulldoc.", ] - if not where_ignored: - gaps.append("unique MATCH WHERE CONTAINS miss did not SET (WHERE might filter)") + if where_ignored: + gaps.append("unique MATCH WHERE CONTAINS miss still SET (WHERE ignored)") if not keyword_ok: gaps.append("find --keyword did not return seeds") - if contains_filters: + where_honoured = not where_ignored + if where_honoured and ret_gate and keyword_ok: verdict = "yes" - elif ret_gate and set_conflict and keyword_ok and where_ignored: - verdict = "note" else: verdict = "no" if not ret_gate: gaps.append("CONTAINS RETURN was not product_gate") - if not set_conflict: - gaps.append("CONTAINS SET was not cue_conflict") + if not where_honoured: + gaps.append("WHERE CONTAINS miss still SET") return ItemResult( item="E17 GQL WHERE CONTAINS substring", verdict=verdict, @@ -2024,13 +2024,14 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: a_exact_4000 = all(a_rows[k].get("exact") for k in keys_4000) a_exact_4096 = all(a_rows[k].get("exact") for k in keys_4096) tab_ok = bool(b_rows["tab_mid"].get("exact") and b_rows["tab_end"].get("exact")) - cr_breaks = not b_rows["cr_mid"].get("exact") and not b_rows["cr_end"].get("exact") + cr_ok = bool(b_rows["cr_mid"].get("exact") and b_rows["cr_end"].get("exact")) + nl_ok = bool(b_rows.get("nl_mid", {}).get("exact")) schema_64_refused = schema_open["64"]["open_exit"] != 0 schema_128_refused = schema_open["128"]["open_exit"] != 0 extras_ram = bool(width64.get("ram_has_extras") and width128.get("ram_has_extras")) - extras_dropped = (not width64.get("loaded_has_extras")) and ( - not width128.get("loaded_has_extras") - ) + extras_save_refused = width64.get("save_exit") not in (0, None) and width128.get( + "save_exit" + ) not in (0, None) fat_ok = bool(fat_row.get("exact")) decoded = bool(caps.get("value_bytes_on_decoded_field") and caps.get("value_bytes_gt_not_ge")) @@ -2042,17 +2043,20 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: gaps.append("SCHEMA 64/128 did not refuse max_fields") if not tab_ok: gaps.append("tab did not round-trip") - if not cr_breaks: - gaps.append("CR unexpectedly round-tripped") + if not cr_ok: + gaps.append("CR did not round-trip") + if not extras_save_refused: + gaps.append("64/128 extras on 4-field USR did not refuse at save (max_fields)") predicted = ( decoded and a_exact_4000 and a_exact_4096 and tab_ok - and cr_breaks + and cr_ok and schema_64_refused and schema_128_refused + and extras_save_refused ) if predicted: verdict = "yes" @@ -2063,10 +2067,11 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: notes = [ "value_bytes 4096 is measured on the decoded field after split_payload " - "(validate_values uses `>` not `>=`). join_payload expands `\\` and `|` " - "only; that expansion is not the cap. line_bytes 32768 is the raw " - "snapshot line before split. SCHEMA register vs max_fields=32. " - "emit_record writes SCHEMA columns only.", + "(check_value_bytes uses `>` not `>=`). join_payload escapes splitlines " + "separators plus `\\` and `|`; that expansion is not the value cap. " + "line_bytes 32768 is the escaped/raw snapshot line. SCHEMA register vs " + "max_fields=32. Snapshot save widens SCHEMA for undeclared RAM keys; " + "extras over max_fields refuse snapshot_unsaveable.", "E18e CJK 4000: " + e18_cjk_composition(4000, han=E18_HAN) + ".", "E18e CJK 4096: " + e18_cjk_composition(4096, han=E18_HAN) + ".", "CREATE wires use gql_str (\\\\ \\' \\\" \\n \\r \\t). Proof numbers omit blobs.", @@ -2089,10 +2094,11 @@ def item_e18(svc: ServeProc, tmp: Path) -> ItemResult: "a_exact_4000": a_exact_4000, "a_exact_4096": a_exact_4096, "tab_roundtrip": tab_ok, - "cr_breaks": cr_breaks, + "cr_roundtrip": cr_ok, + "nl_roundtrip": nl_ok, "schema_64_128_refused": schema_64_refused and schema_128_refused, "extras_in_ram": extras_ram, - "extras_dropped_on_load": extras_dropped, + "extras_save_refused_over_max_fields": extras_save_refused, "fat_8x4000_exact": fat_ok, }, wires=[w for w in wires if w], diff --git a/tests/doc_gate_lib.py b/tests/doc_gate_lib.py index b067573..8b5baa5 100644 --- a/tests/doc_gate_lib.py +++ b/tests/doc_gate_lib.py @@ -33,6 +33,7 @@ MEM_STORE_PY = _ENGINE / "mem_store.py" TAG_MAP_PY = _ENGINE / "tag_map.py" GQL_PY = _ENGINE / "gql.py" +MUTATE_GATE_PY = _ENGINE / "mutate_gate.py" PIN_MAP_INGEST_PY = _ENGINE / "pin_map_ingest.py" PIN_MAP_COMPOSER_PY = _ENGINE / "pin_map_composer.py" CONFIG_PY = _ENGINE / "config.py" @@ -522,35 +523,36 @@ def snapshot_value_cap_report() -> dict[str, Any]: "value_bytes_default": 4096, "line_bytes_default": 32768, "max_fields_default": 32, - "value_bytes_on_decoded_field": "len(val.encode(" in val_src - and "max_value_bytes" in val_src - and ">" in val_src, - "value_bytes_gt_not_ge": 'len(val.encode("utf-8")) > caps.max_value_bytes' in val_src, + "value_bytes_on_decoded_field": "value_utf8_len" in tag and "max_value_bytes" in tag, + "value_bytes_gt_not_ge": "n > caps.max_value_bytes" in tag, "decoded_after_split_payload": ( "values = split_payload(payload)" in parse_src and "validate_values(" in parse_src ), "line_bytes_on_raw_snapshot_line": "max_line_bytes" in parse_src and "len(line.encode(" in parse_src, "join_escapes_backslash_and_pipe": ( - "def join_payload" in join_src - and 'replace("\\\\"' in join_src - and 'replace("|",' in join_src + "def join_payload" in join_src and '"\\\\"' in join_src and '"|"' in join_src ), + "join_escapes_splitlines_separators": "_SPLITLINES_ESC" in wire + and "split_snapshot_lines" in wire, "split_unescapes_before_validate": "split_payload(payload)" in parse_src, "cr_or_nl_is_newline_in_value": '"\\n" in val or "\\r" in val' in val_src, "tab_not_in_newline_check": '"\\t" in val' not in val_src, "max_fields_on_schema_register": "len(field_names) > caps.max_fields" in tag, "emit_record_schema_columns_only": "values = [record.fields.get(f, " in out_src, + "snapshot_widens_undeclared_schema": "_snapshot_emit_tag_map" in snap, "save_write_text_no_value_check": "Path(path).write_text" in snap, "config_value": '_env_int("MEMNET_MAX_VALUE_BYTES", 4096)' in cfg, "config_line": '_env_int("MEMNET_MAX_LINE_BYTES", 32768)' in cfg, "config_fields": '_env_int("MEMNET_MAX_FIELDS", 32)' in cfg, "code_path": ( "session_save -> snapshot_text -> emit_record -> join_payload " - "(escapes \\\\ and | only). session_load -> parse_line: raw line " + "(escapes splitlines separators plus \\\\ and |). session_load -> " + "split_snapshot_lines (LF only) then parse_line: raw escaped line " "vs max_line_bytes, then split_payload unescape, then " - "validate_values decoded utf-8 vs max_value_bytes (`>` not `>=`); " - "CR/LF -> newline_in_value. SCHEMA register vs max_fields." + "validate_values decoded utf-8 vs max_value_bytes (`>` not `>=`). " + "Undeclared RAM keys widen snapshot SCHEMA. SCHEMA register vs " + "max_fields." ), } @@ -781,7 +783,10 @@ def e18_largest_roundtrip( def e18_instance_width(svc: ServeProc, tmp: Path, n_props: int) -> dict[str, Any]: - """n_props keys on 4-field USR SCHEMA. Extras live in RAM; save drops them.""" + """n_props keys on 4-field USR SCHEMA. + + Extras stay in RAM; save widens SCHEMA or refuses max_fields. + """ nid = f"USR_w{n_props}" extra_n = max(0, n_props - 4) extras = {f"x{i:03d}": "v" for i in range(extra_n)} @@ -797,18 +802,24 @@ def e18_instance_width(svc: ServeProc, tmp: Path, n_props: int) -> dict[str, Any public["n_requested"] = n_props public["extra_n"] = extra_n public["ram_has_extras"] = row.get("ram_has_extras") - public["loaded_has_extras"] = row.get("loaded_has_extras") + public["loaded_has_extras"] = row.get("loaded_has_extras", False) + public["save_exit"] = row.get("save_exit") + public["save_err"] = row.get("save_err") + public["load_exit"] = row.get("load_exit") return public def mutate_byte_cap_report() -> dict[str, Any]: - """Pipe leftover caps vs GQL mutate (cap-contract bug 4).""" + """Pipe leftover caps vs GQL mutate (shared decoded value_bytes).""" tag = TAG_MAP_PY.read_text(encoding="utf-8") gql = GQL_PY.read_text(encoding="utf-8") + gate = MUTATE_GATE_PY.read_text(encoding="utf-8") cfg = CONFIG_PY.read_text(encoding="utf-8") cli = CLI_PY.read_text(encoding="utf-8") out = OUTPUT_PY.read_text(encoding="utf-8") composer = PIN_MAP_COMPOSER_PY.read_text(encoding="utf-8") + gql_value = "check_value_bytes" in gate + gql_line = "max_line_bytes" in gql or "max_line_bytes" in gate return { "gql_escapes": r"""\\ \' \" \n \r \t""", "gql_unknown_escape": "unknown string escape" in gql, @@ -817,11 +828,11 @@ def mutate_byte_cap_report() -> dict[str, Any]: "pipe_batch_lines_default": 1000, "pipe_value_code": "limit_exceeded|value_bytes {n}/{max} (inner | -> space on wire)", "pipe_line_code": "limit_exceeded|line_bytes {n}/{max}", - "pipe_newline_code": "newline_in_value", + "pipe_newline_code": "escaped splitlines separators", "pipe_field_count_code": "FIELD_COUNT", "pipe_enforces_in_parse_line": "max_value_bytes" in tag and "max_line_bytes" in tag, - "gql_mutate_checks_value_bytes": "max_value_bytes" in gql, - "gql_mutate_checks_line_bytes": "max_line_bytes" in gql, + "gql_mutate_checks_value_bytes": gql_value, + "gql_mutate_checks_line_bytes": gql_line, "cli_batch_lines": "max_batch_lines" in cli and "batch_lines|" in cli, "wire_pipes_become_spaces": 'message.replace("|", " ")' in out, "locator_equality_only": 'if str(rec.fields.get(key, "")) != val:' in composer, @@ -830,7 +841,7 @@ def mutate_byte_cap_report() -> dict[str, Any]: ), "config_value_bytes": '_env_int("MEMNET_MAX_VALUE_BYTES", 4096)' in cfg, "config_line_bytes": '_env_int("MEMNET_MAX_LINE_BYTES", 32768)' in cfg, - "bug4_gql_skips_pipe_caps": True, + "bug4_gql_skips_pipe_caps": not gql_value, } diff --git a/tests/test_doc_gate_readiness.py b/tests/test_doc_gate_readiness.py index c013690..ba67fb4 100644 --- a/tests/test_doc_gate_readiness.py +++ b/tests/test_doc_gate_readiness.py @@ -189,7 +189,7 @@ def test_snapshot_roundtrip_unicode_pipe_quote(doc_serve: ServeProc, tmp_path: P def test_snapshot_multiline_value_breaks_load(doc_serve: ServeProc, tmp_path: Path): - """Gap: leftover snapshot emit does not escape newlines in field values.""" + """Newlines in a property escape on emit and round-trip (MN-REQ-01.9).""" sid = _open_ok(doc_serve) blob = "Line one.\nLine two." mut = doc_serve.mutate( @@ -202,9 +202,16 @@ def test_snapshot_multiline_value_breaks_load(doc_serve: ServeProc, tmp_path: Pa assert save.exit_code == 0, redact(save.stderr) doc_serve.close(sid) load = doc_serve.load_file(snap) - assert load.exit_code != 0 - joined = "\n".join(err_lines(load.stderr)) - assert "FIELD_COUNT" in joined + assert load.exit_code == 0, redact(load.stderr) + new = None + for line in load.stdout.splitlines(): + if line.startswith("@SESSION:"): + new = line.split("|", 1)[0].replace("@SESSION:", "").strip() + assert new + props = shaped_node_props(doc_serve.pin_map(new, cue="USR_nl").stdout) + assert props is not None + assert props.get("value") == blob + doc_serve.close(new) assert_sid_free(redact(load.stderr), redact(load.stdout)) @@ -252,11 +259,11 @@ def test_acl_who_denied_scope_and_skipped_lifecycle(doc_serve: ServeProc, tmp_pa snap = tmp_path / "acl.snap" save = doc_serve.save(sid, snap) - assert save.exit_code == 0, redact(save.stderr) + assert save.exit_code != 0 + assert any(e.startswith("@ERR: acl_who|") for e in err_lines(save.stderr)) save_caller = doc_serve.save(sid, tmp_path / "acl2.snap", caller="owner") - assert save_caller.exit_code != 0 - assert not any(e.startswith("@ERR: acl_") for e in err_lines(save_caller.stderr)) + assert save_caller.exit_code == 0, redact(save_caller.stderr) bind_mut = doc_serve.mutate( sid, @@ -266,7 +273,10 @@ def test_acl_who_denied_scope_and_skipped_lifecycle(doc_serve: ServeProc, tmp_pa assert bind_mut.exit_code == 0, redact(bind_mut.stderr) assert not any("acl_bind" in e for e in err_lines(bind_mut.stderr)) - closed = doc_serve.close(sid) + closed_who = doc_serve.close(sid) + assert closed_who.exit_code != 0 + assert any(e.startswith("@ERR: acl_who|") for e in err_lines(closed_who.stderr)) + closed = doc_serve.close(sid, caller="owner") assert closed.exit_code == 0, redact(closed.stderr) @@ -372,9 +382,9 @@ def test_mutate_byte_cap_report_bug4(): report = mutate_byte_cap_report() assert report["pipe_value_bytes_default"] == 4096 assert report["pipe_line_bytes_default"] == 32768 - assert report["gql_mutate_checks_value_bytes"] is False + assert report["gql_mutate_checks_value_bytes"] is True assert report["gql_mutate_checks_line_bytes"] is False - assert report["bug4_gql_skips_pipe_caps"] is True + assert report["bug4_gql_skips_pipe_caps"] is False def test_special_blob_has_required_glyphs(): @@ -465,39 +475,33 @@ def test_e13_16kib_ram_roundtrip_snapshot_refused(doc_serve: ServeProc, tmp_path sid, "CREATE (:USR {id: 'USR_big', key: 'blob', value: " + gql_str(blob) + ", recycle: ''})\n", ) - assert create.exit_code == 0, redact(create.stderr) + assert create.exit_code != 0 + joined_c = "\n".join(err_lines(create.stderr)) + assert "value_bytes" in joined_c + assert "16384/4096" in joined_c setted = doc_serve.mutate( sid, "MATCH (n:USR {id: 'USR_big'}) SET n.value = " + gql_str(blob) + "\n", ) - assert setted.exit_code == 0, redact(setted.stderr) - pin = doc_serve.pin_map(sid, cue="USR_big") - assert pin.exit_code == 0, redact(pin.stderr) - props = shaped_node_props(pin.stdout) - assert props is not None - assert props.get("value") == blob + assert setted.exit_code != 0 + assert "value_bytes" in "\n".join(err_lines(setted.stderr)) or "not_found" in "\n".join( + err_lines(setted.stderr) + ) snap = tmp_path / "e13.snap" save = doc_serve.save(sid, snap) assert save.exit_code == 0, redact(save.stderr) doc_serve.close(sid) - load = doc_serve.load_file(snap) - assert load.exit_code != 0 - joined = "\n".join(err_lines(load.stderr)) - assert "FIELD_COUNT" in joined or "value_bytes" in joined or "newline_in_value" in joined - assert "ingest_budget" not in joined sid2 = _open_ok(doc_serve) plain = make_special_blob(16 * 1024, newlines=False, pipes=False) - doc_serve.mutate( + create2 = doc_serve.mutate( sid2, "CREATE (:USR {id: 'USR_p', key: 'blob', value: " + gql_str(plain) + ", recycle: ''})\n", ) - snap2 = tmp_path / "e13p.snap" - doc_serve.save(sid2, snap2) + assert create2.exit_code != 0 + assert "value_bytes" in "\n".join(err_lines(create2.stderr)) + assert "16384/4096" in "\n".join(err_lines(create2.stderr)) doc_serve.close(sid2) - load2 = doc_serve.load_file(snap2) - assert load2.exit_code != 0 - assert "value_bytes" in "\n".join(err_lines(load2.stderr)) def test_e14_list_store_no_in_membership(doc_serve: ServeProc): @@ -523,7 +527,7 @@ def test_e14_list_store_no_in_membership(doc_serve: ServeProc): after = shaped_node_props(doc_serve.pin_map(sid, cue="USR_cite").stdout) or {} miss = shaped_node_props(doc_serve.pin_map(sid, cue="USR_miss").stdout) or {} membership = in_mut.exit_code == 0 and after.get("key") == "hit" and miss.get("key") != "hit" - assert membership is False + assert membership is True, redact(in_mut.stderr) leftover = doc_serve.read_list(sid, tag="USR", where="citeKeys=*k*") assert leftover.exit_code == 0, redact(leftover.stderr) doc_serve.close(sid) @@ -687,22 +691,26 @@ def test_e17_contains_is_not_a_filter(doc_serve: ServeProc): setted = doc_serve.mutate( sid, "MATCH (n:USR) WHERE n.value CONTAINS '测例' SET n.key = 'hit'\n" ) - assert setted.exit_code != 0 - assert "cue_conflict" in "\n".join(err_lines(setted.stderr)) + assert setted.exit_code == 0, redact(setted.stderr) + hit = shaped_node_props(doc_serve.pin_map(sid, cue="USR_a").stdout) or {} + other = shaped_node_props(doc_serve.pin_map(sid, cue="USR_b").stdout) or {} + assert hit.get("key") == "hit" + assert other.get("key") == "m" starts = doc_serve.mutate( - sid, "MATCH (n:USR) WHERE n.value STARTS WITH '测' SET n.key = 'hit'\n" + sid, "MATCH (n:USR) WHERE n.value STARTS WITH '测' SET n.key = 'started'\n" ) - assert "cue_conflict" in "\n".join(err_lines(starts.stderr)) - regex = doc_serve.mutate(sid, "MATCH (n:USR) WHERE n.value =~ '.*测.*' SET n.key = 'hit'\n") - assert "cue_conflict" in "\n".join(err_lines(regex.stderr)) + assert starts.exit_code == 0, redact(starts.stderr) + regex = doc_serve.mutate(sid, "MATCH (n:USR) WHERE n.value =~ '.*测.*' SET n.key = 're'\n") + assert regex.exit_code == 0, redact(regex.stderr) miss = doc_serve.mutate( sid, "MATCH (n:USR {id: 'USR_a'}) WHERE n.value CONTAINS 'ZZZ_NO_MATCH' SET n.key = 'ignored'\n", ) - assert miss.exit_code == 0, redact(miss.stderr) + assert miss.exit_code != 0 + assert "not_found" in "\n".join(err_lines(miss.stderr)) props = shaped_node_props(doc_serve.pin_map(sid, cue="USR_a").stdout) assert props is not None - assert props.get("key") == "ignored" + assert props.get("key") != "ignored" found = doc_serve.find(sid, kind="USR", keyword="测例", limit=10) assert found.exit_code == 0, redact(found.stderr) assert "测例" in found.stdout @@ -723,10 +731,12 @@ def test_snapshot_value_cap_is_decoded_field(): assert report["decoded_after_split_payload"] is True assert report["line_bytes_on_raw_snapshot_line"] is True assert report["join_escapes_backslash_and_pipe"] is True - assert report["cr_or_nl_is_newline_in_value"] is True + assert report["join_escapes_splitlines_separators"] is True + assert report["cr_or_nl_is_newline_in_value"] is False assert report["tab_not_in_newline_check"] is True assert report["max_fields_on_schema_register"] is True assert report["emit_record_schema_columns_only"] is True + assert report["snapshot_widens_undeclared_schema"] is True def test_e18_cjk_blob_composition(): @@ -763,10 +773,9 @@ def test_e18_tab_survives_cr_breaks(doc_serve: ServeProc, tmp_path: Path): cr = e18_roundtrip(doc_serve, tmp_path, blob="ab\rcd", nid="USR_cr") assert cr["create_exit"] == 0, cr assert cr["save_exit"] == 0, cr - assert cr["load_exit"] != 0 - joined = "\n".join(cr["load_err"]) - assert "newline_in_value" in joined or "FIELD_COUNT" in joined - assert_sid_free(joined) + assert cr["load_exit"] == 0, cr["load_err"] + assert cr["exact"] is True + assert_sid_free("\n".join(cr.get("load_err") or [])) def test_e18_4000_backslash_and_pipe_roundtrip(doc_serve: ServeProc, tmp_path: Path): @@ -804,5 +813,12 @@ def test_e18_8x4000_and_ram_extras(doc_serve: ServeProc, tmp_path: Path): assert fat["snap_line_utf8"] < 32768 wide = e18_instance_width(doc_serve, tmp_path, 64) assert wide["ram_has_extras"] is True + assert wide["save_exit"] != 0 + joined = "\n".join(wide.get("save_err") or []) + assert "snapshot_unsaveable" in joined or "fields|" in joined assert wide["loaded_has_extras"] is False - assert wide["load_exit"] == 0, wide + fit = e18_instance_width(doc_serve, tmp_path, 8) + assert fit["ram_has_extras"] is True + assert fit["save_exit"] == 0, fit + assert fit["load_exit"] == 0, fit + assert fit["loaded_has_extras"] is True From f38746c9c60e542cc3def4fe717af47de295f7f9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 14:43:16 +0000 Subject: [PATCH 09/12] Pass --caller on ServeProc.close; honour documented relationship DELETE in E16. Co-authored-by: chouswei --- docs/operations/one-session-per-document.md | 2 +- scripts/probe_doc_gate_readiness.py | 5 ++--- tests/doc_gate_lib.py | 15 ++++++++------- tests/test_doc_gate_readiness.py | 10 ++++++---- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/operations/one-session-per-document.md b/docs/operations/one-session-per-document.md index 1eb071f..8e32eb6 100644 --- a/docs/operations/one-session-per-document.md +++ b/docs/operations/one-session-per-document.md @@ -90,7 +90,7 @@ E16 (on the 10000 fulldoc session, this VM `Intel(R) Xeon(R) Processor` 4-core K | (a) atomic SET 2 KiB + delete 1 edge + add 2 | 115.686 / **133.181** / 155.571 | under bar | | (b) reverse `pin_map` hub `M=400` | 113.096 / **129.613** / 163.101 | under bar | -**note:** MemNet has **no** native delete-refused-while-referenced check. `DETACH DELETE` of a node with inbound edges exits 0 and leaves dangling edges. The gate must refuse from the reverse lookup (`## Truncation truncated=true M=400 omitted=2604 reason=max_rows` on the hub). Documented `MATCH ()-[r {id}]-() DELETE r` is lowered as a node DROP with an empty id and refuses `@ERR: not_found|DELETE matched no element` (not a referenced-delete check). The probe's working edge DROP is `MATCH (n WHERE true)-[r {id}]->() DELETE r`. +**note:** MemNet has **no** native delete-refused-while-referenced check. `DETACH DELETE` of a node with inbound edges exits 0 and leaves dangling edges. The gate must refuse from the reverse lookup (`## Truncation truncated=true M=400 omitted=2604 reason=max_rows` on the hub). `MATCH ()-[r {id}]-() DELETE r` honours relationship DELETE. `MATCH (n WHERE true)-[r {id}]->() DELETE r` remains a valid spelling. E18: **yes.** Snapshot `value_bytes` 4096 is the **decoded** UTF-8 after `split_payload` (`check_value_bytes`: decoded length `>` cap, so 4096 passes). `join_payload` expansion of splitlines separators / `\\` / `|` is not the value cap (4000 `\\` or `|` emit 8000 escaped bytes, snap line ~8021, still loads). `parse_line` measures the escaped/raw line vs `line_bytes` 32768 first; save verify uses the same check. SCHEMA register vs `max_fields=32`. Snapshot save widens SCHEMA for undeclared RAM keys; leftover `emit_record` still writes SCHEMA columns. Loopback CREATE → save → load into a fresh session → `pin_map` cue. Binary search skipped (4000 exact for `\\` and `|`). diff --git a/scripts/probe_doc_gate_readiness.py b/scripts/probe_doc_gate_readiness.py index ee18b91..3ca19bf 100644 --- a/scripts/probe_doc_gate_readiness.py +++ b/scripts/probe_doc_gate_readiness.py @@ -1697,9 +1697,8 @@ def atomic_batch(i: int) -> str: ) gaps.append("no native referenced-delete refuse") notes.append( - "Documented MATCH ()-[r {id}]-() DELETE r lowers as a node DROP with empty id " - "and refuses @ERR: not_found|DELETE matched no element. Atomic (a) uses " - "MATCH (n WHERE true)-[r {id}]->() DELETE r, which reaches EdgeRec DROP." + "MATCH ()-[r {id}]-() DELETE r honours relationship DELETE. Atomic (a) also " + "uses MATCH (n WHERE true)-[r {id}]->() DELETE r." ) if a_fail: gaps.append(f"{a_fail} atomic mutate failures") diff --git a/tests/doc_gate_lib.py b/tests/doc_gate_lib.py index 8b5baa5..966e547 100644 --- a/tests/doc_gate_lib.py +++ b/tests/doc_gate_lib.py @@ -327,12 +327,10 @@ def edge_create(rel: str, eid: str, src: str, dst: str) -> str: def edge_delete(eid: str) -> str: - """Product edge DROP that actually reaches EdgeRec on 0.19.18. + """Product edge DROP that reaches EdgeRec. - Documented ``MATCH ()-[r {id}]-() DELETE r`` lowers as a node DROP with an - empty id and refuses ``@ERR: not_found|DELETE matched no element``. A node - WHERE filter makes ``_parse_node_patterns`` fail, so lowering takes the - relationship-DELETE path. GraphGlot still accepts this form. + ``MATCH ()-[r {id}]-() DELETE r`` now honours the relationship DELETE. + ``MATCH (n WHERE true)-[r {id}]->() DELETE r`` remains a valid spelling. """ return f"MATCH (n WHERE true)-[r {{id: {gql_str(eid)}}}]->() DELETE r" @@ -951,8 +949,11 @@ def try_open_session(self, **kwargs: Any) -> tuple[str | None, ServeReply]: return None, reply return extract_sid(reply.stdout), reply - def close(self, sid: str) -> ServeReply: - return self.send(["session", "close", sid]) + def close(self, sid: str, *, caller: str | None = None) -> ServeReply: + args = ["session", "close", sid] + if caller: + args.extend(["--caller", caller]) + return self.send(args) def live_count(self) -> tuple[int, int]: reply = self.send(["session", "list"]) diff --git a/tests/test_doc_gate_readiness.py b/tests/test_doc_gate_readiness.py index ba67fb4..026c67f 100644 --- a/tests/test_doc_gate_readiness.py +++ b/tests/test_doc_gate_readiness.py @@ -648,10 +648,12 @@ def test_e16_delete_not_refused_while_referenced(doc_serve: ServeProc): assert pin.exit_code == 0, redact(pin.stderr) assert "contains" in pin.stdout documented = doc_serve.mutate(sid, "MATCH ()-[r {id: 'E_drop'}]-() DELETE r\n") - assert documented.exit_code != 0 - joined_doc = "\n".join(err_lines(documented.stderr)) - assert "not_found" in joined_doc - assert "DELETE matched no element" in joined_doc + assert documented.exit_code == 0, redact(documented.stderr) + recreate = doc_serve.mutate( + sid, + edge_create("contains", "E_drop", "SEC_0001", "SEC_0003") + "\n", + ) + assert recreate.exit_code == 0, redact(recreate.stderr) gone = doc_serve.mutate(sid, "MATCH (n:SEC {id: 'SEC_0002'}) DETACH DELETE n\n") assert gone.exit_code == 0, redact(gone.stderr) assert not any(e.startswith("@ERR:") for e in err_lines(gone.stderr)) From 0069d7da97ea4ed4e15757f5a28e6bbe58928029 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 15:00:42 +0000 Subject: [PATCH 10/12] Keep sessions live when expire-save cannot write a snapshot. Expire-save failure (snapshot_unsaveable or OSError) no longer drops RAM. The session still counts against MEMNET_MAX_SESSIONS, retries warn expire_snapshot_failed, and access after TTL is session_expired|overdue until an explicit successful save or close. Save-on-expire off is unchanged. Co-authored-by: chouswei --- CHANGELOG.md | 2 +- docs/cap-contract.md | 8 +- docs/operations/admin-usage-report.md | 7 +- docs/operations/one-session-per-document.md | 4 +- parts/common/memnet/memnet/admin_usage.py | 10 +- parts/common/memnet/memnet/cli.py | 17 +- parts/common/memnet/memnet/output.py | 25 +- parts/common/memnet/memnet/registry.py | 7 + parts/common/memnet/memnet/session.py | 93 +++++-- .../memnet-mcp/software/memnet_mcp/server.py | 18 +- sysml-models/models/behaviour.sysml | 9 +- sysml-models/models/deploy.sysml | 29 ++- sysml-models/models/requirements.sysml | 33 ++- sysml-models/models/verify.sysml | 17 +- tests/cap_contract_lib.py | 3 +- tests/test_admin_usage.py | 2 + tests/test_doc_gate_readiness.py | 1 + tests/test_engine_roundtrip_where_acl.py | 7 +- tests/test_expire_save_keep.py | 242 ++++++++++++++++++ tests/test_mcp.py | 1 + tests/test_snapshot.py | 1 + tests/test_sysml_engine_bugs.py | 3 + tests/test_sysml_expire_save.py | 1 + 23 files changed, 474 insertions(+), 66 deletions(-) create mode 100644 tests/test_expire_save_keep.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3eb84..9426520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. - **Honesty `c` — one session per document over serve (probe)** — Loopback `memnet-serve` readiness for a product gate that keeps one session per document (TTL 60, save-on-expire, `MEMNET_MAX_SESSIONS=1024`, ~1 800 parts). Extra probes E11–E14, E12 on a fulldoc-with-edges fixture (3000 nodes + 4500 edges at 5000 and 10000), E16 latency, E17 `WHERE CONTAINS`, E18 snapshot `value_bytes` (decoded vs escaped) / tab-CR / `max_fields` vs `line_bytes`, and RSS fixtures. Probe, tests, and [`docs/operations/one-session-per-document.md`](docs/operations/one-session-per-document.md) now match the engine fix (lossless snapshot, honoured WHERE, ACL who on save/load/close). No SemVer bump. ### Fixed -- **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes every `str.splitlines()` separator (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus `|`/`\`; load splits records on LF only. Undeclared RAM properties persist by widening the snapshot SCHEMA (live SCHEMA unchanged; extras on EDG/LAW or over `max_fields` fail closed). `line_bytes` is the escaped/raw line at save verify and load. Save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot round-trip writes no file and RAM still drops (`snap_missing`). GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. +- **Honesty `c` — snapshot lossless round-trip, value cap, WHERE, ACL who (#201 follow-on)** — Snapshot emit escapes every `str.splitlines()` separator (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus `|`/`\`; load splits records on LF only. Undeclared RAM properties persist by widening the snapshot SCHEMA (live SCHEMA unchanged; extras on EDG/LAW or over `max_fields` fail closed). `line_bytes` is the escaped/raw line at save verify and load. Save fails closed (`snapshot_unsaveable`) rather than write an unloadable file. Expire-save that cannot write (`snapshot_unsaveable` or any other save failure) writes no file and **keeps RAM**; the session still counts against `MEMNET_MAX_SESSIONS`, retries emit `@WRN: expire_snapshot_failed|{code}`, and access after TTL is `session_expired|overdue` until an explicit successful save or close. When save-on-expire is off, TTL still drops RAM. GQL mutate, leftover pipe, and snapshot load share one decoded `MEMNET_MAX_VALUE_BYTES` cap (`limit_exceeded|value_bytes n/max`). MATCH WHERE SET/DELETE honours the predicate or refuses `unsupported_predicate` with nothing applied. Session save/load/close who-check when ACL is enabled (`--caller` / `MEMNET_CALLER`; MCP passes `caller`). 0.19.18 snapshots still load. MN-REQ-01.9 / 01.10 / 03.4 / 05.3. No version bump. ### Changed - **Invent only — ClusterRoute vs SliceHandCarry (#191 / #47 cousin)** — `MemNetTwoMoves` outside `MemNetSystem` (`MN-REQ-06.9` + `MN-REQ-06.10` / `MN-VER-06-S08`). ClusterRoute = where the session lives (`MemNetLanMcpFront`; one owner; `pin_map` / `find` SHALL NOT span backends). SliceHandCarry = explicit copy into another session (`export_pin_map` or `session_save` → LAN file copy → dest import/`session_load`; `import_slice` same-serve only). Not a live hop. `import_slice(from_url)` not shipped. tip≠face. `inventOnly=true`; `implemented=false`; no engine code; no SemVer bump. Wire: [`docs/operations/cluster-route-vs-slice-hand-carry.md`](docs/operations/cluster-route-vs-slice-hand-carry.md). diff --git a/docs/cap-contract.md b/docs/cap-contract.md index b00b30d..f082d59 100644 --- a/docs/cap-contract.md +++ b/docs/cap-contract.md @@ -161,7 +161,7 @@ Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, **Undeclared properties.** GQL mutate may store keys that are absent from the live tag SCHEMA (GraphElement extras; Path-B locators such as `qname`). Those keys stay in RAM. Snapshot save persists them by widening the **snapshot** SCHEMA (live session SCHEMA is unchanged). After load, the restored map includes the extra columns. Extras on fixed tags `EDG` / `LAW`, or a widened SCHEMA over `max_fields`, refuse `snapshot_unsaveable` and write no file. -Snapshot save verifies every emitted row can parse back to the same values. If any row cannot, save refuses `@ERR: snapshot_unsaveable|{tag} nick={nick} …` and writes no file. Expire-save in that case emits `@WRN: expire_snapshot_failed|snapshot_unsaveable`, writes no file, then RAM still drops (`session_expired|snap_missing`). Snapshots written by 0.19.18 (pipe and backslash escapes only) still load. +Snapshot save verifies every emitted row can parse back to the same values. If any row cannot, save refuses `@ERR: snapshot_unsaveable|{tag} nick={nick} …` and writes no file. Expire-save in that case (or any other save failure, such as an unwritable disk or directory) emits `@WRN: expire_snapshot_failed|{code}` on every sweep or access that retries expiry, writes no file, and **keeps the session in RAM**. It still counts against `MEMNET_MAX_SESSIONS`. Access after TTL is `@ERR: session_expired|overdue`. An explicit `session save` that succeeds, or an explicit `session close`, ends that hold. When save-on-expire is off, TTL still drops RAM. Snapshots written by 0.19.18 (pipe and backslash escapes only) still load. ## Mutate batch line cap @@ -295,11 +295,11 @@ Source: `memnet/catalog_snap.py` `snap_model` / `_precheck_plan`; `memnet/serve. | Expire, save off (default) | Session dropped from memory. First access of the still-registered expired id: `@ERR: session_expired\|snap_missing` (exit 2). After purge already ran: `@ERR: session_not_found\|unknown session` | | Expire, `MEMNET_SAVE_ON_EXPIRE` truthy + `MEMNET_EXPIRE_SNAPSHOT_DIR` set | Snapshot `{dir}/{sid}.snap` (filename only; do not log it). Next use: `@ERR: session_expired\|snap_available`. Restore: `session_load` with that id | | Save-on-expire on, dir unset | `@WRN: save_on_expire_no_dir\|dir unset`, then drop; `snap_missing` | -| Save-on-expire on, row not round-trippable | `@WRN: expire_snapshot_failed\|snapshot_unsaveable`, **no file**, then drop; `snap_missing` | +| Save-on-expire on, row not round-trippable or write fails | `@WRN: expire_snapshot_failed\|{code}`, **no file**, RAM **stays**; access `@ERR: session_expired\|overdue`. Still counts against `MEMNET_MAX_SESSIONS`. Cleared by a successful explicit `session save` or `session close` | | `session save` after TTL with save-on-expire | Allowed; `@WRN: session_expired_saved`. Id then gone | | `session save` after TTL with save off | `@ERR: session_expired` / `snap_missing`; no file | -| Unsaveable explicit save | `@ERR: snapshot_unsaveable\|{tag} nick={nick} …`; no file | -| Status (no paths, no ids) | `@STAT: save_on_expire\|0\|` / `1`; `@STAT: expire_snapshot_dir_set\|0\|` / `1` | +| Unsaveable explicit save | `@ERR: snapshot_unsaveable\|{tag} nick={nick} …`; no file; overdue hold stays if expire-save had already failed | +| Status (no paths, no ids) | `@STAT: save_on_expire\|0\|` / `1`; `@STAT: expire_snapshot_dir_set\|0\|` / `1`; `@STAT: expire_snapshot_failed\|n\|` | Source: `memnet/session.py`, `memnet/config.py` `save_on_expire` / `expire_snapshot_dir`. diff --git a/docs/operations/admin-usage-report.md b/docs/operations/admin-usage-report.md index d153351..0cdbcf9 100644 --- a/docs/operations/admin-usage-report.md +++ b/docs/operations/admin-usage-report.md @@ -57,7 +57,7 @@ A refuse MUST NOT look like a complete empty report (`ok` JSON). If a measured f ```json { "ok": true, - "sessions": {"live": 2, "max": 1024}, + "sessions": {"live": 2, "max": 1024, "expire_snapshot_failed": 0}, "session_rows": [ { "alias": "s_ab12cd34ef56a1b2", @@ -68,7 +68,8 @@ A refuse MUST NOT look like a complete empty report (`ok` JSON). If a measured f "relations_max": 200, "last_access": "2026-10-08T02:00:00Z", "ttl_left_s": 3510, - "save_on_expire_armed": false + "save_on_expire_armed": false, + "expire_snapshot_failed": false } ], "process": { @@ -129,7 +130,7 @@ A refuse MUST NOT look like a complete empty report (`ok` JSON). If a measured f } ``` -`alias` is HMAC-SHA256 of the real sid keyed by `MEMNET_ADMIN_TOKEN`, hex-truncated, prefixed `s_`. Stable for that serve credential; rotating the token rotates aliases. `save_on_expire_armed` repeats the **process** Caps flag (not a per-session arm today). +`alias` is HMAC-SHA256 of the real sid keyed by `MEMNET_ADMIN_TOKEN`, hex-truncated, prefixed `s_`. Stable for that serve credential; rotating the token rotates aliases. `save_on_expire_armed` repeats the **process** Caps flag (not a per-session arm today). `sessions.expire_snapshot_failed` is how many sessions are held in RAM because expire-save could not write; the per-row boolean matches. Peek only: the report does not `session_open` / close / load / save / mutate / purge or slide TTL. diff --git a/docs/operations/one-session-per-document.md b/docs/operations/one-session-per-document.md index 8e32eb6..908ba51 100644 --- a/docs/operations/one-session-per-document.md +++ b/docs/operations/one-session-per-document.md @@ -47,7 +47,7 @@ Client helper: `memnet.serve.send_command(args, stdin=…, host=…, port=…)`. Anything mutate accepts must save and reload as the same property values. Snapshot emit escapes every Python `str.splitlines()` separator (LF, CR, VT, FF, FS/GS/RS, NEL, LS, PS) plus `\` and `|`; load splits records on LF only. A 16 KiB string is refused at GQL CREATE/SET with `@ERR: limit_exceeded|value_bytes 16384/4096` (one decoded cap with leftover pipe and snapshot load). Unicode, `|`, quotes, CR, and newlines on a value under the cap round-trip. Undeclared RAM keys persist by widening the snapshot SCHEMA; extras over `max_fields` refuse `snapshot_unsaveable`. -Expire: with save-on-expire and a dir, TTL drop writes `{dir}/{sid}.snap` (do not log the name). Next use: `@ERR: session_expired|snap_available`. Restore: `session load --session ` (no `--file`). +Expire: with save-on-expire and a dir, TTL drop writes `{dir}/{sid}.snap` (do not log the name). Next use: `@ERR: session_expired|snap_available`. Restore: `session load --session ` (no `--file`). If that write cannot complete (`snapshot_unsaveable`, unwritable disk or directory, or any other save failure), MemNet does **not** drop the session. It stays in RAM, still counts against `MEMNET_MAX_SESSIONS`, and every sweep or access that retries expiry emits `@WRN: expire_snapshot_failed|{code}`. Access after TTL is `@ERR: session_expired|overdue` until an explicit `session save` succeeds or `session close`. `session expire-status`, `session list`, and the admin usage report expose `@STAT: expire_snapshot_failed|n|` / `sessions.expire_snapshot_failed`. When save-on-expire is off, TTL still drops RAM. ## ACL @@ -55,7 +55,7 @@ CapsPolicy is off until grant/enable. Who and WorkerWriteScope apply to `pin_map ## Memory figures -Admin `memnet admin usage-report` (not agent MCP) reports **process** `rss_bytes`. `housekeep stats` is per-session row/edge/orphan counts, not bytes. `session expire-status` is flags only. Measure per-document RSS from `/proc//statm` by subtracting before/after populate. +Admin `memnet admin usage-report` (not agent MCP) reports **process** `rss_bytes` and `sessions.expire_snapshot_failed` (sessions held because expire-save failed). `housekeep stats` is per-session row/edge/orphan counts, not bytes. `session expire-status` is flags plus that hold count. Measure per-document RSS from `/proc//statm` by subtracting before/after populate. The usage report `sessions.live` count is `registry_count()` and can include expired-but-unswept entries. `session list` purges first. Do not treat usage `live` as the true live set until that is fixed. Not fixed in the 0.19.18 probe. diff --git a/parts/common/memnet/memnet/admin_usage.py b/parts/common/memnet/memnet/admin_usage.py index ec46463..1466697 100644 --- a/parts/common/memnet/memnet/admin_usage.py +++ b/parts/common/memnet/memnet/admin_usage.py @@ -22,7 +22,7 @@ from memnet.config import Caps, expire_save_status, serve_max_frame_bytes from memnet.exceptions import MemNetError from memnet.registry import count as registry_count -from memnet.registry import list_entries +from memnet.registry import count_expire_snapshot_failed, list_entries ENV_ADMIN_TOKEN = "MEMNET_ADMIN_TOKEN" ERR_UNCONFIGURED = "admin_unconfigured" @@ -159,10 +159,15 @@ def build_report(token: str) -> dict[str, Any]: if rss is None: unavailable.append("rss_bytes") live = registry_count() + expire_failed = count_expire_snapshot_failed() session_rows = _peek_session_rows(token, caps, flags["save_on_expire"], unavailable) report = { "ok": True, - "sessions": {"live": live, "max": caps.max_sessions}, + "sessions": { + "live": live, + "max": caps.max_sessions, + "expire_snapshot_failed": expire_failed, + }, "session_rows": session_rows, "process": { "version": __version__, @@ -295,6 +300,7 @@ def _peek_session_rows( "last_access": last, "ttl_left_s": ttl_left, "save_on_expire_armed": save_on_expire, + "expire_snapshot_failed": bool(entry.expire_save_failed), } rows.append(row) rows.sort(key=lambda r: str(r.get("alias") or "")) diff --git a/parts/common/memnet/memnet/cli.py b/parts/common/memnet/memnet/cli.py index 5be5c5d..dd1db35 100644 --- a/parts/common/memnet/memnet/cli.py +++ b/parts/common/memnet/memnet/cli.py @@ -63,14 +63,16 @@ _session_is_expired, close_session, count_sessions, + expire_hold_count, + expire_save_should_drop, get_session, + get_session_for_close, get_session_for_save, list_sessions, open_session, purge_expired, resolve_expire_load_path, resolve_session_id, - snapshot_expired_session, ) from memnet.snapshot import load_snapshot, write_snapshot from memnet.tag_map import example_ingest_line @@ -400,8 +402,9 @@ def session_current( def session_list() -> None: """List live session ids (named strata; not ANN) with ``sessions|n/max``.""" caps = _caps() - n = count_sessions() + n = count_sessions(caps) emit_stdout(f"@STAT: sessions|{n}/{caps.max_sessions}") + emit_stat("expire_snapshot_failed", expire_hold_count()) for sid, exp, left, modified in list_sessions(caps): emit_session(sid, exp, str(left), modified) @@ -482,8 +485,10 @@ def session_load( ss = get_session(sid, caps) else: if entry is not None: - snapshot_expired_session(sid, caps) - remove_entry(sid) + if expire_save_should_drop(sid, caps): + remove_entry(sid) + else: + raise MemNetError("session_expired", "overdue", exit_code=2) path = resolve_expire_load_path(sid, caps) ss = load_snapshot( path, @@ -527,8 +532,10 @@ def admin_usage_report( def session_expire_status() -> None: """Booleans for expire-save (serve_status). Path redacted; no sids.""" flags = expire_save_status() + purge_expired(_caps()) emit_stat("save_on_expire", int(flags["save_on_expire"])) emit_stat("expire_snapshot_dir_set", int(flags["expire_snapshot_dir_set"])) + emit_stat("expire_snapshot_failed", expire_hold_count()) @session_app.command("close") @@ -540,7 +547,7 @@ def session_close( ] = None, ) -> None: try: - ss = get_session(session_id, _caps()) + ss = get_session_for_close(session_id, _caps()) _acl_check(ss, caller, "mutate") close_session(session_id, _caps()) emit_session(session_id, "closed") diff --git a/parts/common/memnet/memnet/output.py b/parts/common/memnet/memnet/output.py index 30e632e..60c1cb8 100644 --- a/parts/common/memnet/memnet/output.py +++ b/parts/common/memnet/memnet/output.py @@ -32,11 +32,18 @@ def format_err(code: str, message: str, example: str | None = None) -> str: return f"@ERR: {code}|{msg}" -def format_wrn(code: str, message: str, example: str | None = None) -> str | None: +def format_wrn( + code: str, + message: str, + example: str | None = None, + *, + force: bool = False, +) -> str | None: global _WARN_EMITTED - if _WARN_EMITTED >= _MAX_WRN: - return None - _WARN_EMITTED += 1 + if not force: + if _WARN_EMITTED >= _MAX_WRN: + return None + _WARN_EMITTED += 1 msg = message.replace("|", " ") if example: return f"@WRN: {code}|{msg}|{example}" @@ -47,8 +54,14 @@ def emit_err(error: MemNetError) -> None: emit_stderr(format_err(error.code, error.message, error.example)) -def emit_wrn(code: str, message: str, example: str | None = None) -> None: - line = format_wrn(code, message, example) +def emit_wrn( + code: str, + message: str, + example: str | None = None, + *, + force: bool = False, +) -> None: + line = format_wrn(code, message, example, force=force) if line: emit_stderr(line) diff --git a/parts/common/memnet/memnet/registry.py b/parts/common/memnet/memnet/registry.py index 468b298..b3409f2 100644 --- a/parts/common/memnet/memnet/registry.py +++ b/parts/common/memnet/memnet/registry.py @@ -27,6 +27,8 @@ class SessionEntry: lock: threading.RLock = field(default_factory=threading.RLock) acl: SessionAcl | None = None reserves: NeighbourhoodReserveTable | None = None + # Last expire-save failure code while RAM is held past TTL; None if not held. + expire_save_failed: str | None = None def ensure_reserves(self) -> NeighbourhoodReserveTable: from memnet.neighbourhood_reserve import NeighbourhoodReserveTable @@ -61,6 +63,11 @@ def count() -> int: return len(_sessions) +def count_expire_snapshot_failed() -> int: + with _registry_lock: + return sum(1 for entry in _sessions.values() if entry.expire_save_failed) + + def list_entries() -> list[SessionEntry]: with _registry_lock: return list(_sessions.values()) diff --git a/parts/common/memnet/memnet/session.py b/parts/common/memnet/memnet/session.py index 0caf62b..a7df6a6 100644 --- a/parts/common/memnet/memnet/session.py +++ b/parts/common/memnet/memnet/session.py @@ -27,6 +27,7 @@ SessionEntry, clear_all, count, + count_expire_snapshot_failed, get_entry, list_entries, list_expired_ids, @@ -217,12 +218,33 @@ def resolve_expire_load_path(session_id: str, caps: Caps | None = None) -> Path: raise MemNetError("session_expired", "snap_missing", exit_code=2) +def _mark_expire_save_failed(session_id: str, code: str) -> None: + entry = get_entry(session_id) + if entry is None: + return + entry.expire_save_failed = code + emit_wrn("expire_snapshot_failed", code, force=True) + + +def _clear_expire_save_failed(session_id: str) -> None: + entry = get_entry(session_id) + if entry is not None: + entry.expire_save_failed = None + + +def expire_save_enabled(caps: Caps | None = None) -> bool: + if caps is not None: + return bool(getattr(caps, "save_on_expire", False)) + return save_on_expire() + + def snapshot_expired_session(session_id: str, caps: Caps | None = None) -> str | None: """Configurable expire ``session_save``. Off unless ``MEMNET_SAVE_ON_EXPIRE``. Auto path also needs ``MEMNET_EXPIRE_SNAPSHOT_DIR``. Entry must remain. + On write failure the session stays live and ``expire_save_failed`` is set. """ - enabled = bool(getattr(caps, "save_on_expire", False)) if caps is not None else save_on_expire() + enabled = expire_save_enabled(caps) if not enabled: return None dest_dir = getattr(caps, "expire_snapshot_dir", None) if caps is not None else None @@ -235,39 +257,60 @@ def snapshot_expired_session(session_id: str, caps: Caps | None = None) -> str | return None name = expire_snap_filename(session_id) if name is None: - emit_wrn("expire_snapshot_failed", "unsafe_sid") + _mark_expire_save_failed(session_id, "unsafe_sid") return None from memnet.snapshot import write_snapshot - dest_dir.mkdir(parents=True, exist_ok=True) + try: + dest_dir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + _mark_expire_save_failed(session_id, type(exc).__name__) + return None path = dest_dir / name ss = SessionStore(session_id, caps) try: write_snapshot(ss, path) except OSError as exc: - emit_wrn("expire_snapshot_failed", type(exc).__name__) + _mark_expire_save_failed(session_id, type(exc).__name__) return None except MemNetError as exc: - emit_wrn("expire_snapshot_failed", exc.code) + _mark_expire_save_failed(session_id, exc.code) return None + _clear_expire_save_failed(session_id) emit_wrn("expire_snapshot", "written") return str(path) +def expire_save_should_drop(session_id: str, caps: Caps | None = None) -> bool: + """Try expire-save. True means drop RAM; False means keep live after a write failure.""" + if not expire_save_enabled(caps): + return True + path = snapshot_expired_session(session_id, caps) + if path is not None: + return True + entry = get_entry(session_id) + return not (entry is not None and entry.expire_save_failed) + + def purge_expired(caps: Caps | None = None, *, keep: str | None = None) -> None: now = utc_now() for sid in list_expired_ids(now): if keep is not None and sid == keep: continue - snapshot_expired_session(sid, caps) - remove_entry(sid) + if expire_save_should_drop(sid, caps): + remove_entry(sid) -def count_sessions() -> int: - purge_expired() +def count_sessions(caps: Caps | None = None) -> int: + purge_expired(caps) return count() +def expire_hold_count() -> int: + """Sessions kept in RAM because expire-save failed. Peek; does not purge.""" + return count_expire_snapshot_failed() + + def open_session( map_lines: list[str] | None = None, map_file: str | None = None, @@ -277,10 +320,10 @@ def open_session( ) -> SessionStore: caps = caps or Caps() purge_expired(caps) - if count_sessions() >= caps.max_sessions: + if count_sessions(caps) >= caps.max_sessions: raise MemNetError( "limit_exceeded", - f"sessions|{count_sessions() + 1}/{caps.max_sessions}", + f"sessions|{count_sessions(caps) + 1}/{caps.max_sessions}", ) if ttl_minutes is None: ttl_minutes = default_ttl_minutes() @@ -329,10 +372,12 @@ def get_session(session_id: str, caps: Caps | None = None) -> SessionStore: raise_session_miss(session_id, caps, saw_expire=False) expires = datetime.fromisoformat(entry.meta.expires_at.replace("Z", "+00:00")) if expires < utc_now(): - snapshot_expired_session(session_id, caps) - remove_entry(session_id) - purge_expired(caps) - raise_session_miss(session_id, caps, saw_expire=True) + if expire_save_should_drop(session_id, caps): + remove_entry(session_id) + purge_expired(caps) + raise_session_miss(session_id, caps, saw_expire=True) + purge_expired(caps, keep=session_id) + raise MemNetError("session_expired", "overdue", exit_code=2) # Sliding TTL: extend on access (avoids silent expiry for long sessions) original_ttl = entry.meta.ttl_minutes new_expires = utc_now() + timedelta(minutes=original_ttl) @@ -375,18 +420,30 @@ def list_sessions(caps: Caps | None = None) -> list[tuple[str, str, int, str]]: out: list[tuple[str, str, int, str]] = [] for entry in list_entries(): expires = datetime.fromisoformat(entry.meta.expires_at.replace("Z", "+00:00")) - if expires < now: + overdue = expires < now + if overdue and not entry.expire_save_failed: continue - ttl_left = max(0, int((expires - now).total_seconds() // 60)) + ttl_left = 0 if overdue else max(0, int((expires - now).total_seconds() // 60)) modified = entry.meta.modified_at or "-" out.append((entry.meta.session_id, entry.meta.expires_at, ttl_left, modified)) out.sort(key=lambda row: row[0]) return out +def get_session_for_close(session_id: str, caps: Caps | None = None) -> SessionStore: + """Load a session for ``session_close``, including overdue expire-save holds.""" + caps = caps or Caps() + entry = get_entry(session_id) + if entry is None: + purge_expired(caps) + raise_session_miss(session_id, caps, saw_expire=False) + purge_expired(caps, keep=session_id) + return SessionStore(session_id, caps) + + def close_session(session_id: str, caps: Caps | None = None) -> None: caps = caps or Caps() - ss = get_session(session_id, caps) + ss = get_session_for_close(session_id, caps) with ss.lock(exclusive=True): if not remove_entry(session_id): raise MemNetError("session_not_found", "unknown session", exit_code=2) diff --git a/parts/memnet-mcp/software/memnet_mcp/server.py b/parts/memnet-mcp/software/memnet_mcp/server.py index d6a7231..3bfdec9 100644 --- a/parts/memnet-mcp/software/memnet_mcp/server.py +++ b/parts/memnet-mcp/software/memnet_mcp/server.py @@ -50,14 +50,22 @@ async def _run(argv: list[str], *, stdin: str | None = None, session: str | None return _json(resp) -def _expire_flags_from_stat(stdout: str) -> dict[str, bool]: - flags = expire_save_status() +def _expire_flags_from_stat(stdout: str) -> dict[str, bool | int]: + flags: dict[str, bool | int] = dict(expire_save_status()) + from memnet.session import expire_hold_count + + flags["expire_snapshot_failed"] = expire_hold_count() for line in stdout.splitlines(): stripped = line.strip() if stripped.startswith("@STAT: save_on_expire|"): flags["save_on_expire"] = stripped.split("|", 2)[1] == "1" elif stripped.startswith("@STAT: expire_snapshot_dir_set|"): flags["expire_snapshot_dir_set"] = stripped.split("|", 2)[1] == "1" + elif stripped.startswith("@STAT: expire_snapshot_failed|"): + try: + flags["expire_snapshot_failed"] = int(stripped.split("|", 2)[1]) + except ValueError: + pass return flags @@ -68,7 +76,10 @@ async def serve_status() -> str: Also reports whether expire-save is armed (booleans only; path redacted). When TCP serve is up, flags come from the serve process. """ - flags = expire_save_status() + from memnet.session import expire_hold_count + + flags: dict[str, bool | int] = dict(expire_save_status()) + flags["expire_snapshot_failed"] = expire_hold_count() running = probe() if running: raw = send_command(["session", "expire-status"]) @@ -80,6 +91,7 @@ async def serve_status() -> str: "port": serve_port(), "save_on_expire": flags["save_on_expire"], "expire_snapshot_dir_set": flags["expire_snapshot_dir_set"], + "expire_snapshot_failed": int(flags.get("expire_snapshot_failed") or 0), } ) diff --git a/sysml-models/models/behaviour.sysml b/sysml-models/models/behaviour.sysml index 832da44..f1b0aa5 100644 --- a/sysml-models/models/behaviour.sysml +++ b/sysml-models/models/behaviour.sysml @@ -92,9 +92,12 @@ package MemNetBehaviour { } attribute def EvExpireTtl { doc /* - Sliding TTL elapsed. RAM drop. Optional SnapshotStore file only - when saveOnExpire is configured (MEMNET_SAVE_ON_EXPIRE). File stays - until the user deletes it. Not Neo4j. + Sliding TTL elapsed. RAM drop unless expire-save is on and the + snapshot cannot be written: then RAM stays, counts against the + session cap, and retries warn expire_snapshot_failed until an + explicit successful save or close. Optional SnapshotStore file + only when saveOnExpire is configured (MEMNET_SAVE_ON_EXPIRE). + File stays until the user deletes it. Not Neo4j. */ } attribute def EvCloseOrExpire { diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index e8b5c42..d4681f0 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -1064,11 +1064,13 @@ package MemNet { (MN-REQ-01.4--01.6). MUST NOT be taken as MN-REQ-11 pin-map export. Expire-time save is configurable (MEMNET_SAVE_ON_EXPIRE default off). When on, TTL drops RAM and MAY write a file - (MEMNET_EXPIRE_SNAPSHOT_DIR and/or session save --file). The file - remains until the user deletes it. session_load restores a live - session. Explicit session_save and expire-save write one blob - (explicitSaveAndExpireSaveOneBlob). MUST NOT treat this as Neo4j - cabinet dump. storageRole file_snapshot. + (MEMNET_EXPIRE_SNAPSHOT_DIR and/or session save --file) unless + the write fails: then RAM stays (expireSaveFailureKeepsRam), + still counts against maxSessions, and retries warn + expire_snapshot_failed. The file remains until the user deletes + it. session_load restores a live session. Explicit session_save + and expire-save write one blob (explicitSaveAndExpireSaveOneBlob). + MUST NOT treat this as Neo4j cabinet dump. storageRole file_snapshot. */ port snapOut : SessionSnapshotOutPort; port snapIn : SessionSnapshotInPort; @@ -1089,6 +1091,12 @@ package MemNet { attribute losslessPropertyRoundTrip : Boolean = true; attribute failClosedUnsaveable : Boolean = true; attribute expireSaveUnsaveableNoFile : Boolean = true; + attribute expireSaveFailureKeepsRam : Boolean = true; + attribute expireSaveFailureCountsCap : Boolean = true; + attribute expireSaveFailureWarnRetry : Boolean = true; + attribute expireSaveFailureOverdueAccess : Boolean = true; + attribute expireSaveFailureClearedBySaveOrClose : Boolean = true; + attribute expireSaveOffDropsRam : Boolean = true; attribute legacy01918Load : Boolean = true; attribute escapeLfCrPipeBackslash : Boolean = true; attribute splitlinesSeparatorsEscaped : Boolean = true; @@ -1141,6 +1149,7 @@ package MemNet { part snap : SnapshotStore; attribute slidingTtl : Boolean = true; attribute ramDropsOnTtl : Boolean = true; + attribute expireSaveFailureKeepsRam : Boolean = true; connection gqlParseToMutate : GraphRecordFlow { end port source ::> gqlWire.recordsOut; @@ -1393,7 +1402,9 @@ package MemNet { part def CmdSessionSave { doc /* TARGET: session save. Session handle is not a graph store key. - After TTL, save only if MEMNET_SAVE_ON_EXPIRE; then RAM id drops. + After TTL, save only if MEMNET_SAVE_ON_EXPIRE; then RAM id drops + when the snapshot writes. A failed expire-save keeps RAM until + an explicit successful session_save or session_close. */ attribute requiresStoreKeyId : Boolean = false; attribute implemented : Boolean = true; @@ -1401,6 +1412,7 @@ package MemNet { attribute saveOnExpireConfigurable : Boolean = true; attribute saveOnExpireDefault : Boolean = false; attribute ramDropsAfterExpireSave : Boolean = true; + attribute expireSaveFailureKeepsRam : Boolean = true; attribute acceptsCaller : Boolean = true; attribute aclWhoWhenEnabled : Boolean = true; attribute failClosedUnsaveable : Boolean = true; @@ -2959,11 +2971,14 @@ package MemNet { part def ServeStatusLook { doc /* Display serve_status: up / host / port plus expire-save booleans - (save_on_expire, expire_snapshot_dir_set). Path redacted. Look only. + (save_on_expire, expire_snapshot_dir_set) and the count of + sessions held because expire-save failed + (expire_snapshot_failed). Path redacted. Look only. */ attribute lookOnly : Boolean = true; attribute saveOnExpireFlag : Boolean = true; attribute expireSnapshotDirSetFlag : Boolean = true; + attribute expireSnapshotFailedCount : Boolean = true; attribute pathRedacted : Boolean = true; } diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index da9f358..ed00482 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -26,7 +26,8 @@ MN-REQ-06.8 Tip MemNet access portal (ops): admin invite, Google login, Bearer for keyed tip MCP; tip≠face MN-REQ-06.9 LAN MCP front invent (#191): ClusterRoute — where the session lives; one MCP catalogue, N LAN serves; cousin of #47; inventOnly MN-REQ-06.10 SliceHandCarry invent (#47 cousin): bounded WorkingMemorySlice copy into another session; not a live hop; inventOnly - MN-REQ-01.9 Snapshot save/load lossless property round-trip; fail-closed save + MN-REQ-01.9 Snapshot save/load lossless property round-trip; fail-closed save; + expire-save failure keeps RAM and counts against the session cap MN-REQ-01.10 CapsPolicy who-check on session save / load-into-ACL / close MN-REQ-03.4 MATCH WHERE SET/DELETE SHALL honour the predicate or refuse MN-REQ-05.3 One decoded value_bytes cap on pipe mutate, GQL mutate, snapshot load @@ -148,11 +149,15 @@ package MemNetRequirements { doc /* SHALL apply configurable session TTL and a cap on concurrent sessions. Sliding TTL: live access extends expires_at. After - TTL the in-memory session SHALL drop. Optional expire - session_save is configurable (MEMNET_SAVE_ON_EXPIRE, default - off): file snapshot MAY remain on disk until the user deletes - it; session_load restores RAM. SHALL NOT auto-flush Neo4j. - Explicit session_save and this expire-save write one + TTL the in-memory session SHALL drop, except when expire-save + is on and the snapshot cannot be written: RAM SHALL stay + (MN-REQ-01.9) and the session still counts against + MEMNET_MAX_SESSIONS. Optional expire session_save is + configurable (MEMNET_SAVE_ON_EXPIRE, default off): file + snapshot MAY remain on disk until the user deletes it; + session_load restores RAM. When save-on-expire is off, TTL + drops RAM as before. SHALL NOT auto-flush Neo4j. Explicit + session_save and this expire-save write one SessionSnapshotBlob. */ attribute requirementId : String = "MN-REQ-01.3"; @@ -228,9 +233,19 @@ package MemNetRequirements { naming the row) rather than write an unloadable file, including when extras would exceed max_fields or the escaped line exceeds line_bytes. Expire-save that cannot - round-trip SHALL NOT write a file; RAM still drops; next - use is session_expired|snap_missing. Snapshots written by - 0.19.18 (pipe and backslash escapes only) SHALL still load. + write (snapshot_unsaveable or any other save failure, + such as an unwritable disk or directory) SHALL NOT write + a file and SHALL NOT drop RAM. The session stays live, + still counts against MEMNET_MAX_SESSIONS, and every sweep + or access that retries expiry SHALL emit + @WRN: expire_snapshot_failed|. Access after TTL + SHALL report the session overdue + (session_expired|overdue) so the caller can fix and save, + or close. An explicit session_save that succeeds, or an + explicit session_close, ends that hold. When + MEMNET_SAVE_ON_EXPIRE is off, TTL still drops RAM as + MN-REQ-01.3. Snapshots written by 0.19.18 (pipe and + backslash escapes only) SHALL still load. */ attribute requirementId : String = "MN-REQ-01.9"; } diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 1ce8c1f..5d44c66 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -1864,7 +1864,8 @@ package MemNetVerification { separators (LF/CR/VT/FF/FS/GS/RS/NEL/LS/PS) plus pipe/backslash losslessly; undeclared RAM properties persist by snapshot SCHEMA widen; line_bytes is the escaped emitted line at save and load; - save fails closed; expire-save writes no unloadable file; + save fails closed; expire-save writes no unloadable file and + keeps RAM on any save failure until explicit save or close; 0.19.18 snapshots still load; value_bytes is decoded raw UTF-8. */ attribute verificationId : String = "MN-VER-01-S04"; @@ -1881,6 +1882,20 @@ package MemNetVerification { .failClosedUnsaveable == true and system.core.transport.inProcess.memory.sessions.snap .expireSaveUnsaveableNoFile == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveFailureKeepsRam == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveFailureCountsCap == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveFailureWarnRetry == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveFailureOverdueAccess == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveFailureClearedBySaveOrClose == true + and system.core.transport.inProcess.memory.sessions.snap + .expireSaveOffDropsRam == true + and system.core.cli.sessionSave.expireSaveFailureKeepsRam + == true and system.core.transport.inProcess.memory.sessions.snap .legacy01918Load == true and system.core.transport.inProcess.memory.sessions.snap diff --git a/tests/cap_contract_lib.py b/tests/cap_contract_lib.py index 4b0785e..4d62137 100644 --- a/tests/cap_contract_lib.py +++ b/tests/cap_contract_lib.py @@ -1062,7 +1062,8 @@ def case_where_and_snapshot_honesty() -> list[Case]: wire=format_err(exc.code, exc.message), extra={ "expire": ( - "@WRN: expire_snapshot_failed|snapshot_unsaveable then snap_missing" + "@WRN: expire_snapshot_failed|snapshot_unsaveable; " + "RAM stays; @ERR: session_expired|overdue" ), }, ) diff --git a/tests/test_admin_usage.py b/tests/test_admin_usage.py index ff8bf78..fce90b9 100644 --- a/tests/test_admin_usage.py +++ b/tests/test_admin_usage.py @@ -73,6 +73,7 @@ def test_alias_not_real_session_id(memnet_temp, monkeypatch, schema_file): assert report["ok"] is True assert report["sessions"]["live"] == 1 assert report["sessions"]["max"] == Caps().max_sessions + assert report["sessions"]["expire_snapshot_failed"] == 0 rows = report["session_rows"] assert len(rows) == 1 alias = rows[0]["alias"] @@ -85,6 +86,7 @@ def test_alias_not_real_session_id(memnet_temp, monkeypatch, schema_file): assert rows[0]["relations_max"] == Caps().max_relations assert rows[0]["ttl_left_s"] >= 0 assert isinstance(rows[0]["save_on_expire_armed"], bool) + assert rows[0]["expire_snapshot_failed"] is False assert ss.meta.expires_at == expires_before assert "process" in report assert report["process"]["version"] diff --git a/tests/test_doc_gate_readiness.py b/tests/test_doc_gate_readiness.py index 026c67f..53f6a5a 100644 --- a/tests/test_doc_gate_readiness.py +++ b/tests/test_doc_gate_readiness.py @@ -133,6 +133,7 @@ def test_serve_envelope_has_no_mcp_errors_field(doc_serve: ServeProc): stats = stat_lines(reply.stdout) assert any(s.startswith("@STAT: save_on_expire|1|") for s in stats) assert any(s.startswith("@STAT: expire_snapshot_dir_set|1|") for s in stats) + assert any(s.startswith("@STAT: expire_snapshot_failed|") for s in stats) assert_sid_free(redact(reply.stdout), redact(reply.stderr)) diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py index f588756..8806656 100644 --- a/tests/test_engine_roundtrip_where_acl.py +++ b/tests/test_engine_roundtrip_where_acl.py @@ -383,7 +383,10 @@ def test_explicit_save_unsaveable_names_row(memnet_temp, schema_file, tmp_path: def test_expire_save_unsaveable_writes_no_file( memnet_temp, schema_file, tmp_path: Path, monkeypatch ): - """If round-trip verify fails, expire-save warns and leaves no snap file.""" + """If round-trip verify fails, expire-save warns, writes no file, keeps RAM.""" + from memnet.registry import contains + from memnet.session import expire_hold_count + monkeypatch.setenv("MEMNET_SAVE_ON_EXPIRE", "1") monkeypatch.setenv("MEMNET_EXPIRE_SNAPSHOT_DIR", str(tmp_path)) ss = open_session(map_file=str(schema_file), caps=Caps()) @@ -393,3 +396,5 @@ def test_expire_save_unsaveable_writes_no_file( snapshot_expired_session(ss.session_id, Caps()) snaps = list(tmp_path.glob("*.snap")) assert snaps == [] + assert contains(ss.session_id) + assert expire_hold_count() == 1 diff --git a/tests/test_expire_save_keep.py b/tests/test_expire_save_keep.py new file mode 100644 index 0000000..7c7a065 --- /dev/null +++ b/tests/test_expire_save_keep.py @@ -0,0 +1,242 @@ +"""MN-REQ-01.9 — expire-save failure keeps the session live until save or close.""" + +from __future__ import annotations + +import os +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from memnet.cli import app +from memnet.config import Caps +from memnet.exceptions import MemNetError +from memnet.mutate_gate import MutateGate +from memnet.registry import contains +from memnet.session import ( + count_sessions, + expire_hold_count, + get_session, + open_session, + purge_expired, + set_now_override, +) + +runner = CliRunner() +_PLR = ( + "CREATE (:PLR {id: 'PLR01', identity: 'Hero', wealth: 1, cashflow: 0, " + "monopoly: 0, reputation: 0, inventory: 'bag'})" +) +_TINY_MAP = ["SCHEMA N ; fields=id name"] +_TTL_WAIT_S = 62 + + +def _make_unsaveable(ss) -> None: + MutateGate(ss).apply([_PLR], mode="add") + rec = ss.store.get("PLR01") + rec.fields["identity"] = "x" * 9000 + + +def _arm_expire(monkeypatch, tmp_path: Path) -> Caps: + monkeypatch.setenv("MEMNET_SAVE_ON_EXPIRE", "1") + monkeypatch.setenv("MEMNET_EXPIRE_SNAPSHOT_DIR", str(tmp_path)) + return Caps() + + +def _expire_now() -> None: + set_now_override(datetime.now(UTC) + timedelta(minutes=5)) + + +def test_unsaveable_expiry_keeps_session_and_warns( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + caps = _arm_expire(monkeypatch, tmp_path) + ss = open_session(map_file=str(schema_file), ttl_minutes=1, caps=caps) + sid = ss.session_id + _make_unsaveable(ss) + _expire_now() + pin = runner.invoke(app, ["query", "pin-map", "--cue", "PLR01", "--session", sid]) + assert pin.exit_code == 2 + assert "session_expired" in pin.stderr + assert "overdue" in pin.stderr + assert "expire_snapshot_failed" in pin.stderr + assert "snapshot_unsaveable" in pin.stderr + assert contains(sid) + assert expire_hold_count() == 1 + assert list(tmp_path.glob("*.snap")) == [] + status = runner.invoke(app, ["session", "expire-status"]) + assert "@STAT: expire_snapshot_failed|1|" in status.stdout + listed = runner.invoke(app, ["session", "list"]) + assert "@STAT: expire_snapshot_failed|1|" in listed.stdout + set_now_override(None) + + +def test_unsaveable_expiry_later_save_clears_hold( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + caps = _arm_expire(monkeypatch, tmp_path) + ss = open_session(map_file=str(schema_file), ttl_minutes=1, caps=caps) + sid = ss.session_id + _make_unsaveable(ss) + _expire_now() + purge_expired(caps) + assert contains(sid) + assert expire_hold_count() == 1 + ss.store.get("PLR01").fields["identity"] = "Hero" + dest = tmp_path / "fixed.snap" + save = runner.invoke(app, ["session", "save", "--file", str(dest), "--session", sid]) + assert save.exit_code == 0, save.stderr + assert dest.is_file() + assert not contains(sid) + assert expire_hold_count() == 0 + status = runner.invoke(app, ["session", "expire-status"]) + assert "@STAT: expire_snapshot_failed|0|" in status.stdout + set_now_override(None) + + +def test_unsaveable_expiry_close_clears_hold(memnet_temp, schema_file, tmp_path: Path, monkeypatch): + caps = _arm_expire(monkeypatch, tmp_path) + ss = open_session(map_file=str(schema_file), ttl_minutes=1, caps=caps) + sid = ss.session_id + _make_unsaveable(ss) + _expire_now() + with pytest.raises(MemNetError) as exc: + get_session(sid, caps) + assert exc.value.code == "session_expired" + assert exc.value.message == "overdue" + assert contains(sid) + closed = runner.invoke(app, ["session", "close", sid]) + assert closed.exit_code == 0, closed.stderr + assert "closed" in closed.stdout + assert not contains(sid) + assert expire_hold_count() == 0 + set_now_override(None) + + +def test_unsaveable_expiry_counts_against_session_cap( + memnet_temp, schema_file, tmp_path: Path, monkeypatch +): + monkeypatch.setenv("MEMNET_MAX_SESSIONS", "1") + caps = _arm_expire(monkeypatch, tmp_path) + ss = open_session(map_file=str(schema_file), ttl_minutes=1, caps=caps) + sid = ss.session_id + _make_unsaveable(ss) + _expire_now() + purge_expired(caps) + assert contains(sid) + assert count_sessions(caps) == 1 + with pytest.raises(MemNetError) as exc: + open_session(map_file=str(schema_file), ttl_minutes=1, caps=Caps()) + assert exc.value.code == "limit_exceeded" + assert "sessions|" in exc.value.message + set_now_override(None) + + +def test_unwritable_dir_expiry_keeps_session(memnet_temp, schema_file, tmp_path: Path, monkeypatch): + expire_dir = tmp_path / "expire" + expire_dir.mkdir() + caps = _arm_expire(monkeypatch, expire_dir) + ss = open_session(map_file=str(schema_file), ttl_minutes=1, caps=caps) + sid = ss.session_id + MutateGate(ss).apply([_PLR], mode="add") + os.chmod(expire_dir, 0o555) + try: + _expire_now() + pin = runner.invoke(app, ["query", "pin-map", "--cue", "PLR01", "--session", sid]) + assert pin.exit_code == 2 + assert "overdue" in pin.stderr + assert "expire_snapshot_failed" in pin.stderr + assert contains(sid) + assert expire_hold_count() == 1 + assert list(expire_dir.glob("*.snap")) == [] + finally: + os.chmod(expire_dir, 0o755) + set_now_override(None) + + +def test_save_on_expire_off_still_drops(memnet_temp, schema_file, monkeypatch): + monkeypatch.delenv("MEMNET_SAVE_ON_EXPIRE", raising=False) + ss = open_session(map_file=str(schema_file), ttl_minutes=1) + sid = ss.session_id + _expire_now() + with pytest.raises(MemNetError) as exc: + get_session(sid) + assert exc.value.code == "session_expired" + assert exc.value.message == "snap_missing" + assert not contains(sid) + assert expire_hold_count() == 0 + set_now_override(None) + + +def _extras_create(n: int = 40) -> str: + extras = ", ".join(f"x{i:02d}: 'v'" for i in range(n)) + return f"CREATE (:N {{id: 'N01', name: 'n', {extras}}})" + + +def _wait_ttl() -> None: + time.sleep(_TTL_WAIT_S) + + +def test_live_unsaveable_expiry_keeps_warns_cap_and_close(tmp_path: Path): + from tests.doc_gate_lib import running_serve + + with running_serve(tmp_path, ttl_minutes=1, max_sessions=1) as svc: + sid, opened = svc.try_open_session(map_lines=list(_TINY_MAP), ttl=1) + assert sid, opened.stderr + created = svc.mutate(sid, _extras_create()) + assert created.exit_code == 0, created.stderr + _wait_ttl() + pin = svc.pin_map(sid, cue="N01") + assert pin.exit_code == 2 + assert "session_expired" in pin.stderr + assert "overdue" in pin.stderr + assert "expire_snapshot_failed" in pin.stderr + assert "snapshot_unsaveable" in pin.stderr + n, mx = svc.live_count() + assert n == 1 + assert mx == 1 + status = svc.expire_status() + assert "@STAT: expire_snapshot_failed|1|" in status.stdout + blocked, reply = svc.try_open_session(map_lines=list(_TINY_MAP), ttl=1) + assert blocked is None + assert "limit_exceeded" in reply.stderr + assert "sessions|" in reply.stderr + closed = svc.close(sid) + assert closed.exit_code == 0, closed.stderr + status2 = svc.expire_status() + assert "@STAT: expire_snapshot_failed|0|" in status2.stdout + sid2, opened2 = svc.try_open_session(map_lines=list(_TINY_MAP), ttl=1) + assert sid2, opened2.stderr + assert svc.close(sid2).exit_code == 0 + + +def test_live_unwritable_expiry_save_clears_hold(tmp_path: Path): + from tests.doc_gate_lib import running_serve + + with running_serve(tmp_path, ttl_minutes=1, max_sessions=2) as svc: + sid = svc.open_session(map_lines=list(_TINY_MAP), ttl=1) + created = svc.mutate(sid, "CREATE (:N {id: 'N01', name: 'ok'})") + assert created.exit_code == 0, created.stderr + os.chmod(svc.snap_dir, 0o555) + try: + _wait_ttl() + pin = svc.pin_map(sid, cue="N01") + assert pin.exit_code == 2 + assert "overdue" in pin.stderr + assert "expire_snapshot_failed" in pin.stderr + assert "@STAT: expire_snapshot_failed|1|" in svc.expire_status().stdout + finally: + os.chmod(svc.snap_dir, 0o755) + dest = tmp_path / "fixed.snap" + saved = svc.save(sid, dest) + assert saved.exit_code == 0, saved.stderr + assert dest.is_file() + assert "@STAT: expire_snapshot_failed|0|" in svc.expire_status().stdout + n, _mx = svc.live_count() + assert n == 0 + pin2 = svc.pin_map(sid, cue="N01") + assert pin2.exit_code == 2 + assert "session_expired" in pin2.stderr or "session_not_found" in pin2.stderr + assert "overdue" not in pin2.stderr diff --git a/tests/test_mcp.py b/tests/test_mcp.py index b60876d..132f884 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -127,6 +127,7 @@ def test_serve_status_tool(monkeypatch): assert "port" in payload assert payload["save_on_expire"] is False assert payload["expire_snapshot_dir_set"] is False + assert payload["expire_snapshot_failed"] == 0 def test_query_warm_tool_envelope(memnet_temp, schema_file, monkeypatch): diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index a159f27..5425232 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -396,4 +396,5 @@ def test_session_expire_status_booleans(memnet_temp, tmp_path: Path, monkeypatch assert on.exit_code == 0, on.output assert "@STAT: save_on_expire|1|" in on.stdout assert "@STAT: expire_snapshot_dir_set|1|" in on.stdout + assert "@STAT: expire_snapshot_failed|0|" in on.stdout _assert_err_sid_free(on.stdout, on.stderr) diff --git a/tests/test_sysml_engine_bugs.py b/tests/test_sysml_engine_bugs.py index 3f65092..8a55530 100644 --- a/tests/test_sysml_engine_bugs.py +++ b/tests/test_sysml_engine_bugs.py @@ -16,6 +16,8 @@ def test_requirements_ids_present(): for nid in ("MN-REQ-01.9", "MN-REQ-01.10", "MN-REQ-03.4", "MN-REQ-05.3"): assert nid in text assert "snapshot_unsaveable" in text + assert "expire_snapshot_failed" in text + assert "SHALL NOT drop RAM" in text assert "unsupported_predicate" in text assert "MEMNET_MAX_VALUE_BYTES" in text assert "str.splitlines()" in text @@ -39,6 +41,7 @@ def test_deploy_and_verify_trail_model(): assert "splitlinesSeparatorsEscaped" in deploy assert "recordSplitLfOnly" in deploy assert "lineBytesOnEmittedLine" in deploy + assert "expireSaveFailureKeepsRam" in deploy ver = VERIFY.read_text(encoding="utf-8") assert "MN-VER-01-S04" in ver assert "MN-VER-01-S05" in ver diff --git a/tests/test_sysml_expire_save.py b/tests/test_sysml_expire_save.py index 15707f4..7e9ca2c 100644 --- a/tests/test_sysml_expire_save.py +++ b/tests/test_sysml_expire_save.py @@ -38,6 +38,7 @@ def test_snapshot_store_and_caps_default_off(): assert "attribute loadRestoresRam : Boolean = true;" in text assert "attribute notNeo4j : Boolean = true;" in text assert "attribute ramDropsAfterExpireSave : Boolean = true;" in text + assert "attribute expireSaveFailureKeepsRam : Boolean = true;" in text assert "attribute ttlResetsOnLoad : Boolean = true;" in text assert "attribute loadByKnownSid : Boolean = true;" in text assert "attribute expireSnapKeepId : Boolean = true;" in text From 93b8eb7e019bdec1030cc6661e7b34ce86e80a53 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 15:07:40 +0000 Subject: [PATCH 11/12] Fix live expire-keep cap assert to match wire sessions 2/1. Serve @ERR replaces inner pipes with spaces; the hold still counts against MEMNET_MAX_SESSIONS. Co-authored-by: chouswei --- tests/test_expire_save_keep.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_expire_save_keep.py b/tests/test_expire_save_keep.py index 7c7a065..33fc779 100644 --- a/tests/test_expire_save_keep.py +++ b/tests/test_expire_save_keep.py @@ -10,6 +10,7 @@ import pytest from typer.testing import CliRunner +from doc_gate_lib import running_serve from memnet.cli import app from memnet.config import Caps from memnet.exceptions import MemNetError @@ -180,8 +181,6 @@ def _wait_ttl() -> None: def test_live_unsaveable_expiry_keeps_warns_cap_and_close(tmp_path: Path): - from tests.doc_gate_lib import running_serve - with running_serve(tmp_path, ttl_minutes=1, max_sessions=1) as svc: sid, opened = svc.try_open_session(map_lines=list(_TINY_MAP), ttl=1) assert sid, opened.stderr @@ -202,7 +201,7 @@ def test_live_unsaveable_expiry_keeps_warns_cap_and_close(tmp_path: Path): blocked, reply = svc.try_open_session(map_lines=list(_TINY_MAP), ttl=1) assert blocked is None assert "limit_exceeded" in reply.stderr - assert "sessions|" in reply.stderr + assert "sessions 2/1" in reply.stderr closed = svc.close(sid) assert closed.exit_code == 0, closed.stderr status2 = svc.expire_status() @@ -213,8 +212,6 @@ def test_live_unsaveable_expiry_keeps_warns_cap_and_close(tmp_path: Path): def test_live_unwritable_expiry_save_clears_hold(tmp_path: Path): - from tests.doc_gate_lib import running_serve - with running_serve(tmp_path, ttl_minutes=1, max_sessions=2) as svc: sid = svc.open_session(map_lines=list(_TINY_MAP), ttl=1) created = svc.mutate(sid, "CREATE (:N {id: 'N01', name: 'ok'})") From e126ad69b7861a4d8d5b3a1fc1e3c469b34c9532 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 15:07:40 +0000 Subject: [PATCH 12/12] Test Endleaf MATCH (n WHERE true)-[r {id}]->() DELETE r. Trivial WHERE true stays honoured, not unsupported_predicate. In-process and live serve loopback cover the exact no-space id map spelling the gate sends. Co-authored-by: chouswei --- tests/test_doc_gate_readiness.py | 28 ++++++++++++++++++++++++ tests/test_engine_roundtrip_where_acl.py | 11 ++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/tests/test_doc_gate_readiness.py b/tests/test_doc_gate_readiness.py index 53f6a5a..5edd12b 100644 --- a/tests/test_doc_gate_readiness.py +++ b/tests/test_doc_gate_readiness.py @@ -629,6 +629,34 @@ def test_e12_fulldoc_scaled_write_read_load(tmp_path: Path): assert "rows" in joined +def test_endleaf_where_true_edge_delete(doc_serve: ServeProc): + """Endleaf deletes edges with MATCH (n WHERE true)-[r {id:'…'}]->() DELETE r only.""" + sid = _open_ok(doc_serve) + setup = doc_serve.mutate( + sid, + sec_create(1) + + "\n" + + sec_create(2) + + "\n" + + edge_create("contains", "E_endleaf", "SEC_0001", "SEC_0002") + + "\n", + ) + assert setup.exit_code == 0, redact(setup.stderr) + before = doc_serve.pin_map(sid, cue="SEC_0001", depth=1, max_rows=20) + assert before.exit_code == 0, redact(before.stderr) + assert "contains" in before.stdout + stmt = "MATCH (n WHERE true)-[r {id:'E_endleaf'}]->() DELETE r\n" + gone = doc_serve.mutate(sid, stmt) + assert gone.exit_code == 0, redact(gone.stderr) + joined = "\n".join(err_lines(gone.stderr)) + assert "unsupported_predicate" not in joined + assert "unsupported_predicate" not in gone.stderr + after = doc_serve.pin_map(sid, cue="SEC_0001", depth=1, max_rows=20) + assert after.exit_code == 0, redact(after.stderr) + assert "contains" not in after.stdout + doc_serve.close(sid) + + def test_e16_delete_not_refused_while_referenced(doc_serve: ServeProc): sid = _open_ok(doc_serve) setup = doc_serve.mutate( diff --git a/tests/test_engine_roundtrip_where_acl.py b/tests/test_engine_roundtrip_where_acl.py index 8806656..7a329c9 100644 --- a/tests/test_engine_roundtrip_where_acl.py +++ b/tests/test_engine_roundtrip_where_acl.py @@ -296,6 +296,7 @@ def test_unsupported_where_refuses_nothing_applied(memnet_temp, schema_file): def test_where_true_edge_delete_still_works(memnet_temp, schema_file): + """Endleaf gate spelling: MATCH (n WHERE true)-[r {id:'…'}]->() DELETE r.""" ss = open_session(map_file=str(schema_file)) MutateGate(ss).apply([_PLR], mode="add") MutateGate(ss).apply( @@ -310,10 +311,12 @@ def test_where_true_edge_delete_still_works(memnet_temp, schema_file): mode="add", allow_new_relation=True, ) - MutateGate(ss).apply( - ["MATCH (n WHERE true)-[r {id: 'E_k'}]->() DELETE r"], - mode="mutate", - ) + stmt = "MATCH (n WHERE true)-[r {id:'E_k'}]->() DELETE r" + try: + MutateGate(ss).apply([stmt], mode="mutate") + except MemNetError as ei: + assert ei.value.code != "unsupported_predicate", ei.value.message + raise assert ss.store.get("E_k") is None