diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 435dd26ed8..6583227073 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -255,12 +255,14 @@ jobs: # Each group keeps a hard deadline. The six independent recovery scenarios # must not consume a single shared deadline before client-exit recovery runs. + # One scenario reloads the baseline, upgrades the full Server chain, recovers + # it, and verifies history, so the group budget covers 6 x ~110s plus collection. - name: Recover committed Server DDL and domain checkpoints on real OceanBase without skips env: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + timeout --signal=TERM --kill-after=10s 900s uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-checkpoints.xml @@ -299,8 +301,8 @@ jobs: Path(".powercontext/recall-effort-ci/full-server-maintenance.xml"), ) cases = [case for report in reports for case in ElementTree.parse(report).iter("testcase")] - if len(cases) != 11 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All eleven native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 13 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All thirteen native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index b45d1fc07a..e3ef2f3df8 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -17,8 +17,8 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 The full Server framework uses two normalization checkpoints before its shared migration chain for the PowerContext 1.5.0 target (stored compatibility IDs shown below): -- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001`. -- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001` through a separate frozen source bridge. +- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`. +- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003` through a separate frozen source bridge. Application release numbers and schema revisions are distinct. The 1.4.0 release is not the `s1400` checkpoint: its existing Candidate schema is current, but its Topic attempt storage still needs the registered bridge. @@ -98,6 +98,15 @@ earlier revision resources and checksums remain unchanged. Native vector tables are admitted to later maintenance only after exact layout validation. These registrations do not expand the accepted source versions or establish installed-release upgrade acceptance. +Revision `v1_5_0_002` adds the Source evidence bindings, exact Atomic Memory evidence snapshots, and evidence schema marker. Its own domain step validates retained Source material and historical evidence for both lexical and vector profiles. Earlier projection validators and their recorded revision checksums remain unchanged; evidence validation is required only from this revision onward. Existing Sources and Memory revisions without accepted declarations stay neutral (`untrusted` / `unknown`); upgrading never backfills trust. Current projection repair and rebuild also require evidence validation. + +Revision `v1_5_0_003` registers `pc_atomic_memory_evidence`, the disposable exact-revision evidence projection. It is +derived from retained history, can be discarded and rebuilt, and never becomes the authority for a trust decision. + +The runtime bundle stamps `pc_atomic_memory_evidence_schema.schema_version` with `p0005`, its own revision for the same +tables; the Server chain registers that shape as `v1_5_0_002`. Operator messages name both so a runtime deployment and a +managed Server deployment can each apply the revision they own. + An interrupted [portable restore](portable-archive.md) keeps normal startup and workers blocked until its projections are complete. Retry `powercontext archive restore --yes` with the same validated bundle. Only this explicit recovery path can open an incomplete projection set, after validating the registered layout and the configured vector @@ -144,16 +153,18 @@ Sequence `000` identifies a registered historical baseline; upgrade steps start | `v1_4_1_001` | 1.4.1 | 1 | `s1400` | Complete Server storage transition | | `v1_4_1_002` | 1.4.1 | 2 | `s1401` | Offline Memory history archive | | `v1_4_1_003` | 1.4.1 | 3 | `s1402` | Portable restore receipts and timestamp adaptation | -| `v1_4_1_004` | 1.4.1 | 4 | `v1_4_1_004` | Durable owner recovery intents | +| `v1_4_1_004` | 1.4.1 | 4 | `v1_4_1_004` | Durable external-owner recovery intents | | `v1_4_1_005` | 1.4.1 | 5 | `v1_4_1_005` | Nullable Atomic Memory description projection | | `v1_4_1_006` | 1.4.1 | 6 | `v1_4_1_006` | Source write plugin registrations | | `v1_5_0_001` | 1.5.0 | 1 | `v1_5_0_001` | Artifact plugin storage and search registrations | +| `v1_5_0_002` | 1.5.0 | 2 | `v1_5_0_002` | Retained Source evidence and historical validation | +| `v1_5_0_003` | 1.5.0 | 3 | `v1_5_0_003` | Disposable exact-revision evidence projection | -Both registered source paths reach `v1_5_0_001`; only 1.3.0 is selected for CI upgrade acceptance: +Both registered source paths reach `v1_5_0_003`; only 1.3.0 is selected for CI upgrade acceptance: ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 ``` `status` and `plan` show `source_migration`, `target_migration` and `pending_migrations` @@ -175,7 +186,7 @@ existing plans and original backups usable. Partial prototype bundles keep their For a new Server migration, use the release-based name directly as its Alembic `revision` and register its schema, domain steps and recovery checkpoints as usual. `down_revision` must reference -the actual preceding ID (`v1_5_0_001` when appending directly to the current head), even if that ID has +the actual preceding ID (`v1_5_0_003` when appending directly to the current head), even if that ID has a display alias. The framework follows this registered chain, not lexical name ordering. Already registered names and release assignments stay immutable; a version bump must not rename history. @@ -270,6 +281,11 @@ bound row buffering; this is an offline whole-projection operation. It does not embedded seekDB Code configurations. The compatibility command `server atomic-memory-rebuild-projection` delegates managed databases to this bundle and supports `--action plan/apply/verify` with the same confirmation and backup options. +Both maintenance entry points run from the migration bundle and therefore admit evidence against the builtin Source +registry; they do not receive a host's code-supplied `SourceDefinitionRegistry`. The runtime evidence read path +(`AtomicMemory.evidence`) uses the configured registry, so a deployment with custom native Definitions should rebuild +through the Python API with `sources=` when it needs an identical verdict. + ## Topic Memory recovery bundle Select `--bundle topic-memory-recovery` for the versioned `powercontext.topic-memory-recovery.v1` conversion. It recognizes @@ -723,6 +739,10 @@ SQL literals reach the backend unchanged. Repositories run after schema checks. Both full Server source chains register exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. +During `v1_5_0_002`, an interrupted MySQL-compatible upgrade can resume a registered CREATE-table prefix before the +final `p0005` evidence marker is inserted. Full evidence validation starts once that marker exists; a completed +database with missing or incompatible evidence storage still fails admission. + Online readiness reads Atomic Memory authority and projections from the same snapshot without locking business head rows. Offline projection repair retains its locked state comparison and write-side concurrency protection. The 1.4.0 bridge is assembled from immutable shared resources and its `server_sources/1.4.0` overlay. Source diff --git a/docs/en/docs/operate/observability.md b/docs/en/docs/operate/observability.md index b30f4564de..d354077e47 100644 --- a/docs/en/docs/operate/observability.md +++ b/docs/en/docs/operate/observability.md @@ -65,6 +65,14 @@ including code stages. Code-specific diagnostic fields are included only for `co | `duration_ms`, `error_code`, `error_category` | Duration in milliseconds, diagnostic code, and bounded operational error classification | | `stage`, `processing_stage`, `failure_reason`, `input_item_count`, `estimated_input_tokens`, `token_estimator_id`, `effective_budget`, `provider_request_id`, `finish_reason`, `failure_location`, `cause_exception_type` | Processing failure stage/reason and safe accounting fields; `stage` names the emitting boundary or domain location (Worker/supervisor stage, domain block name, or `worker` fallback), `processing_stage` the bounded domain stage. See [Processing failure diagnostics](../../development/processing-failures.md) | +`reason` is a bounded cause for a retained-evidence downgrade. The `atomic_memory.evidence.neutral` event reports that +accepted evidence could not be attested and therefore resolved to `untrusted` / `unknown`; it never carries material, +declaration bodies, or requested content. Its `unit` is the retained fact (`source`, `declaration`, `manifest`, or +`revision`), `outcome` is `neutral`, and `reason` is one of `schema_mismatch`, `digest_mismatch`, `no_manifest`, +`decode_failed`, `legacy_unverifiable`, `invalid_declaration`, `manifest_conflict`, `definition_missing`, or +`evidence_mismatch`. Integrity causes are emitted at `WARNING`; expected historical gaps stay at `INFO` or `DEBUG`. +Integrity causes such as `manifest_conflict` are alertable and distinct from an absent declaration. + Server roles are `all` (API and supervisor), `api`, and `background`; child processes use `worker`. Explicit logging configuration without a deployment role uses `server`. JSON identity fields correspond to trace resource attributes `service.name`, `service.version`, `service.instance.id`, and `powercontext.service.role`. `service_version` is absent diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index 8a31d1e974..2530552e2f 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -131,6 +131,7 @@ Format version 1 carries the portable relational representation of these support | --- | --- | | Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | | Source journal heads, Source records, and referenced remote Source Definition manifests | Search projections and indexes | +| Accepted Memory evidence: per-Source declaration bindings, the Definition manifests they reference, and exact-revision evidence snapshots | Today's registry declarations, which never replace a saved snapshot | | Artifact Revisions, their timestamps/actors/rollback metadata, stable handles, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | | Atomic Memory merge relationships and marked merge-input lineage | External Skill registrations and host-local installation state | | Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | diff --git a/docs/en/docs/workflows/sources.md b/docs/en/docs/workflows/sources.md index 78de15d770..51090d4a5a 100644 --- a/docs/en/docs/workflows/sources.md +++ b/docs/en/docs/workflows/sources.md @@ -33,4 +33,54 @@ content under the same identity as if it were a new observation. - Subject evidence for a Scope profile: follow [Scope profiles](use-profiles.md). - Application or evaluation events: use the application's adapter; [Bub capture](../integrations/evaluation.md) is opt-in. +## Declare source authority + +A trusted adapter can declare how its material should be trusted. The optional `memory_evidence` field on a Source +Definition carries a versioned `MemoryEvidenceDeclaration` with `authority` (`untrusted`, `user_asserted`, +`repository_attested`, or `system_attested`) and `verification` (`unknown`, `verified`, or `not_verified`). Only +adapter or deployment code supplies it; Source text, references, model output, and retrieval rank cannot. + +```python +from powercontext.sources import AdapterSourceDefinition, MemoryEvidenceDeclaration + +definition = AdapterSourceDefinition( + existing_adapter, + memory_evidence=MemoryEvidenceDeclaration( + authority="repository_attested", + verification="not_verified", + ), +) +``` + +Remote Source observations configure the same declaration in trusted deployment code, keyed by the exact Definition +name, version, and manifest fingerprint. A missing, invalid, or unavailable declaration resolves to +`untrusted`/`unknown`; it does not block capture, processing, or existing retrieval. + +Observation manifests describe the serialized payload for every field the payload must carry. Validation aliases do +not require workers to rename serialized keys, including when a model forbids extra fields. +For unions, an alias-renamed branch may retain the validation manifest when another branch has an equivalent flat +primitive-property schema covering its serialized output. Unproven branch coverage uses the serialization schema, except +for a field whose default is `null`: it can reach the payload as `null`, so a renamed shape below it never replaces an +accepted manifest. Populating such a field with a renamed key requires the nested model to serialize by alias, which +keeps the validation keys authoritative. Changing what a registered Definition serializes requires a new Definition +version. + +Each Memory revision records the declaration and exact Source materialization accepted with it. That retained evidence +is authoritative and versioned. The index projection beside it is derived: maintenance can discard and rebuild it +without consulting today's registry. Declarations add no ranking weight, score, or access right. + +Read the retained facts back for one exact revision: + +```python +from powercontext.builtin.runtime import AtomicMemoryEvidenceAnnotation + +memory = runtime.atomic_memory.for_scope(scope_id) +annotation: AtomicMemoryEvidenceAnnotation = await memory.evidence(artifact_id, revision=3) +for source in annotation.sources: + print(source.source_ref.source_type, source.declaration.authority, source.declaration.verification) +``` + +A declaration that can no longer be attested resolves to `untrusted`/`unknown`. A record that no longer matches its +exact Source material raises `AtomicMemoryEvidenceError`, which callers catch through the `PowerContextError` family. + Source writes and reads are subject to the selected [Scope and access policy](scopes-and-access.md). diff --git a/docs/en/rfcs/1652_memory_quality_and_lifecycle.md b/docs/en/rfcs/1652_memory_quality_and_lifecycle.md index c6e21f4d2f..0deff631b1 100644 --- a/docs/en/rfcs/1652_memory_quality_and_lifecycle.md +++ b/docs/en/rfcs/1652_memory_quality_and_lifecycle.md @@ -222,6 +222,24 @@ candidate pool or delivered Context Pack occupied by a lower-authority class, bu legitimate answer-bearing evidence. This is an opt-in experiment, not a stage-1 default or a substitute for the Source declaration contract. +## Current implementation amendment: source evidence declaration + +The Source evidence prerequisite is implemented as specified above. This section supersedes the "provisionally named" +wording and the statement that `SourceDefinition` carries no authority or verification declaration. + +- `SourceDefinition` and `AdapterSourceDefinition` expose an optional `memory_evidence` declaration with `authority`, + `verification`, and `declaration_version="1"`. +- Trusted adapter code, or deployment configuration keyed by an exact `(name, version, fingerprint)` Definition + manifest, supplies the declaration. Source text, `SourceRef`, model output, and retrieval rank cannot. +- Each Atomic Memory revision retains the accepted declaration with its exact Source materialization and manifest + fingerprint. A separate index-owned projection holds the same facts for that revision and can be discarded and + rebuilt from retained evidence without consulting the current registry. +- Missing, invalid, or unverifiable declarations remain `untrusted`/`unknown` for reads and rebuilds. There is no + trust backfill, and existing write authority, generation eligibility, body text, content hashes, citations, and + search contracts are unchanged. +- Stage-1 quality ranking, lifecycle actions, and receipt-linked usage feedback remain unimplemented. This slice is + their prerequisite, not a substitute for them. + ## Independent quality dimensions The policy keeps these dimensions separate: diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index caa564bd9f..be639972e5 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -14,8 +14,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 完整 Server 框架使用两个归一化检查点接入共享迁移链： -- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001`。 -- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001`。 +- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`。 +- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_4_1_005 → v1_4_1_006 → v1_5_0_001 → v1_5_0_002 → v1_5_0_003`。 应用发布版本与 schema revision 分别编号。1.4.0 发行包不等于 `s1400` 检查点：其 Candidate 结构已经是当前格式， 但 Topic 尝试状态仍需登记的桥接。两条路径保留既有引用和历史，到达相同的受管目标结构；原有资源摘要分别保留， @@ -78,6 +78,15 @@ SQLite 还登记可移植向量和 FTS 辅助结构。两条来源链均通过 根据已登记空间创建的原生向量表，必须通过精确布局检查才能纳入后续维护。上述登记不扩展受支持的来源版本， 也不代表已通过真实安装发布版的升级验收。 +修订 `v1_5_0_002` 新增 Source 证据绑定、精确 Atomic Memory 证据快照和证据结构标记。它使用独立领域步骤验证保留的 Source 材料与历史证据，覆盖词法和向量配置。早期投影验证器及已记录的修订校验和保持不变；仅从此修订开始要求证据验证。既有 Source 和 Memory 修订若未接受声明，仍保持中性（`untrusted` / `unknown`），升级不会回填信任。当前投影修复与重建也必须验证证据。 + +修订 `v1_5_0_003` 登记 `pc_atomic_memory_evidence`，即精确 revision 的可丢弃证据投影。它由保留历史派生， +可以丢弃并重建，从不作为信任判定的权威依据。 + +runtime bundle 在 `pc_atomic_memory_evidence_schema.schema_version` 中写入 `p0005`，即它自身应用同一结构所用的 +revision；Server 链将同一结构登记为 `v1_5_0_002`。运维提示会同时给出两者，使 runtime 部署与受管 Server 部署 +各自应用所属的 revision。 + [可移植归档恢复](portable-archive.md) 中断后，普通启动和 worker 会继续拒绝未完成的检索投影。 请使用同一个通过校验的归档，重新执行 `powercontext archive restore --yes`。 仅此显式恢复路径可在验证已登记结构和配置的向量 profile 后打开投影尚未完整的数据库，且不启动 worker。 @@ -116,16 +125,18 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `v1_4_1_001` | 1.4.1 | 第一步 | `s1400` | 完整 Server 存储转换 | | `v1_4_1_002` | 1.4.1 | 第二步 | `s1401` | 离线 Memory 历史归档 | | `v1_4_1_003` | 1.4.1 | 第三步 | `s1402` | 可移植恢复回执与时间戳适配 | -| `v1_4_1_004` | 1.4.1 | 第四步 | `v1_4_1_004` | 持久化所有权恢复意图 | +| `v1_4_1_004` | 1.4.1 | 第四步 | `v1_4_1_004` | 持久外部所有权恢复意图 | | `v1_4_1_005` | 1.4.1 | 第五步 | `v1_4_1_005` | Atomic Memory 可空摘要投影列 | | `v1_4_1_006` | 1.4.1 | 第六步 | `v1_4_1_006` | Source write 插件注册 | | `v1_5_0_001` | 1.5.0 | 第一步 | `v1_5_0_001` | Artifact 插件存储与检索登记 | +| `v1_5_0_002` | 1.5.0 | 第二步 | `v1_5_0_002` | 保留 Source 证据及历史验证 | +| `v1_5_0_003` | 1.5.0 | 第三步 | `v1_5_0_003` | 可丢弃的精确 revision 证据投影 | -两条已登记来源路径都升级到 `v1_5_0_001`；CI 升级验收只选择 1.3.0： +两条已登记来源路径都升级到 `v1_5_0_003`；CI 升级验收只选择 1.3.0： ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_4_1_005 -> v1_4_1_006 -> v1_5_0_001 -> v1_5_0_002 -> v1_5_0_003 ``` `status` 和 `plan` 在原有机器字段之前显示 `source_migration`、`target_migration`、 @@ -144,7 +155,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 这样可以继续使用已有计划和原始备份。局部原型 bundle 保留自己的独立编号。 后续新增 Server 迁移直接使用该命名作为 Alembic `revision`，并正常登记结构、领域步骤和恢复检查点。 -`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_001`，即使它已有展示别名。 +`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_003`，即使它已有展示别名。 框架按照登记的链执行，不按名称字典序推算。已登记的名称和所属发布版本保持不变，版本升级不能重命名历史。 ## 初始化与维护职责 @@ -228,6 +239,11 @@ apply 只修复派生行，完整 Server 校验通过后才提交。权威数据 兼容命令 `server atomic-memory-rebuild-projection` 在受管数据库上转发到此 bundle， 支持 `--action plan/apply/verify` 和同样的确认、备份选项。 +这两个维护入口均由迁移 bundle 构建，因此证据准入使用内置 Source registry，拿不到宿主以代码传入的 +`SourceDefinitionRegistry`。运行时的证据读取路径（`AtomicMemory.evidence`）使用已配置的 registry； +若部署注册了自定义 native Definition 且需要一致的判定结果，应通过 Python API 传入 +`sources=` 执行重建。 + ## Topic Memory 恢复 bundle 选择 `--bundle topic-memory-recovery` 执行版本化的 `powercontext.topic-memory-recovery.v1` 转换。它识别恢复功能之前的 @@ -528,6 +544,9 @@ MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `m Repository 仅在结构满足后运行。两条完整 Server 来源链均用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 +在 `v1_5_0_002` 中，MySQL 兼容升级若在最终 `p0005` evidence 标记写入前中断，可以从已登记的建表前缀继续。 +标记存在后才要求完整 evidence 校验；已完成的数据库若缺失 evidence 存储或结构不兼容，仍会被准入检查拒绝。 + 在线就绪校验在同一快照中读取 Atomic Memory 权威记录与投影，不锁定业务 head 行。离线投影修复仍保留加锁状态比较和写入并发保护。 1.4.0 桥接由不可改写的共享资源和 `server_sources/1.4.0` 来源覆盖层组装。执行器根据完整未登记版本的数据库 diff --git a/docs/zh/docs/operate/observability.md b/docs/zh/docs/operate/observability.md index a06b56a393..c4165120a9 100644 --- a/docs/zh/docs/operate/observability.md +++ b/docs/zh/docs/operate/observability.md @@ -63,6 +63,14 @@ Code 专用诊断字段仅在 `code.*` events 中输出。 | `duration_ms`、`error_code`、`error_category` | 毫秒耗时、诊断 code 和有限取值的 operational error category | | `stage`、`processing_stage`、`failure_reason`、`input_item_count`、`estimated_input_tokens`、`token_estimator_id`、`effective_budget`、`provider_request_id`、`finish_reason`、`failure_location`、`cause_exception_type` | 处理失败 stage/reason 与安全计数字段；`stage` 表示发出失败的位置或领域位置（Worker/supervisor stage、领域 block 名称或 `worker` 兜底），`processing_stage` 表示有界的领域阶段。详见 [处理失败诊断](../../development/processing-failures.md) | +`reason` 是保留证据降级的有限原因。`atomic_memory.evidence.neutral` 事件表示已接受的证据无法被佐证， +因此解析为 `untrusted` / `unknown`；该事件不携带材料、声明正文或请求内容。其 `unit` 为保留事实的类型 +（`source`、`declaration`、`manifest` 或 `revision`），`outcome` 为 `neutral`，`reason` 取 +`schema_mismatch`、`digest_mismatch`、`no_manifest`、`decode_failed`、`legacy_unverifiable`、 +`invalid_declaration`、`manifest_conflict`、`definition_missing` 或 `evidence_mismatch` 之一。 +已登记的完整性原因以 `WARNING` 输出；可预期的历史缺口保持 `INFO` 或 `DEBUG`。`manifest_conflict` 这类完整性原因 +可告警，并与“从未声明”区分开。 + Server role 为 `all`（API 与 supervisor）、`api` 或 `background`，child process 使用 `worker`。 显式配置 logging 但没有指定 deployment role 时使用 `server`。JSON identity 字段对应 trace resource attributes： `service.name`、`service.version`、`service.instance.id` 和 `powercontext.service.role`。 diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index 8ce6c4707e..00619d77f1 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -111,6 +111,7 @@ powercontext archive restore ./backups/payments.pcb --yes | --- | --- | | Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | | Source journal head、Source record 和引用的远程 Source Definition manifest | 搜索 projection 和 index | +| 已接受的 Memory evidence：逐 Source 的 declaration binding、其引用的 Definition manifest 和精确 revision 的 evidence snapshot | 当前 registry 的 declaration，它不会覆盖已保存的快照 | | Artifact Revision 及其时间戳、actor、回滚元数据；稳定句柄、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | | Atomic Memory 合并关系和带合并输入标记的 lineage | External Skill registration 和 host-local installation state | | Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | diff --git a/docs/zh/docs/workflows/sources.md b/docs/zh/docs/workflows/sources.md index 98f7b7519a..f4aa54661b 100644 --- a/docs/zh/docs/workflows/sources.md +++ b/docs/zh/docs/workflows/sources.md @@ -31,4 +31,51 @@ Work Contract 和 Task Outcome 也会保留 Source 证据。 - Scope Profile 的主体证据：按 [Scope Profile](use-profiles.md)写入。 - 应用或评测事件：使用应用对应适配器；[Bub 采集](../integrations/evaluation.md)需要显式启用。 +## 声明 Source 权威性 + +受信任的 adapter 可以声明其材料的可信程度。Source Definition 上的可选 `memory_evidence` 字段携带版本化的 +`MemoryEvidenceDeclaration`，其中 `authority` 取 `untrusted`、`user_asserted`、`repository_attested` 或 +`system_attested`，`verification` 取 `unknown`、`verified` 或 `not_verified`。它只能由 adapter 或部署代码提供， +Source 正文、引用、模型输出和检索排名都不能。 + +```python +from powercontext.sources import AdapterSourceDefinition, MemoryEvidenceDeclaration + +definition = AdapterSourceDefinition( + existing_adapter, + memory_evidence=MemoryEvidenceDeclaration( + authority="repository_attested", + verification="not_verified", + ), +) +``` + +远程 Source observation 在受信任的部署代码中配置同一声明，键为精确的 Definition 名称、版本与 manifest +fingerprint。声明缺失、无效或不可用时解析为 `untrusted`/`unknown`，不阻塞采集、处理或既有检索。 + +Observation manifest 描述 payload 必须携带的每个字段的序列化形态。即使模型禁止额外字段，validation alias 也不要求 +worker 重命名序列化后的键。 +对于 Union，若另一分支的等价扁平基础字段 schema 能覆盖 alias 分支的序列化输出，可保留 validation manifest。 +无法证明分支覆盖时使用 serialization schema，但默认值为 `null` 的字段除外：该字段在 payload 里可以是 `null`， +其下的改名形态不得替换已接受的 manifest。要让这样的字段以改名后的键出现，嵌套模型必须按 alias 序列化，从而保持 +validation 键的权威性。改变已注册 Definition 的序列化表示需要新的 Definition 版本。 + +每个 Memory revision 记录当时接受的声明与精确 Source materialization。保留的证据是权威且带版本的；其旁的索引 +投影是派生数据，维护可以丢弃它并从保留事实重建，且不查询当前 registry。声明不增加排序权重、不改变分数， +也不授予访问权限。 + +按精确 revision 读回保留事实： + +```python +from powercontext.builtin.runtime import AtomicMemoryEvidenceAnnotation + +memory = runtime.atomic_memory.for_scope(scope_id) +annotation: AtomicMemoryEvidenceAnnotation = await memory.evidence(artifact_id, revision=3) +for source in annotation.sources: + print(source.source_ref.source_type, source.declaration.authority, source.declaration.verification) +``` + +无法再被佐证的声明解析为 `untrusted`/`unknown`。若记录与精确 Source 材料不再一致，会抛出 +`AtomicMemoryEvidenceError`，调用方可通过 `PowerContextError` 家族捕获。 + Source 读写受所选 [Scope 与访问策略](scopes-and-access.md)约束。 diff --git a/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md b/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md index 06dce9a62e..3e1d5779e1 100644 --- a/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md +++ b/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md @@ -195,6 +195,21 @@ Memory entry 写入时，必须把声明版本及解析后的 authority/verifica 进行对比：限制低 authority 类别在固定候选池或最终 Context Pack 中占用的位置/字节份额，但必须保留显式路径，允许 确实能回答问题的证据进入。它是 opt-in 的评测实验，不是第一阶段默认行为，也不能替代 Source 声明 contract。 +## 当前实现修订：Source 证据声明 + +上文要求的 Source 证据前置条件已按此规格实现。本节取代“暂称”的表述，以及 `SourceDefinition` 不携带 +authority/verification 声明的说法。 + +- `SourceDefinition` 与 `AdapterSourceDefinition` 提供可选的 `memory_evidence` 声明，包含 `authority`、 + `verification` 与 `declaration_version="1"`。 +- 声明只能由受信任的 adapter 代码，或在部署配置中以精确 `(name, version, fingerprint)` Definition manifest + 为键提供。Source 正文、`SourceRef`、模型输出与检索排名都不能。 +- 每个 Atomic Memory revision 保留当时接受的声明、精确 Source materialization 与 manifest fingerprint。同一 + revision 的相同事实另有索引拥有的派生投影；该投影可丢弃，并由保留证据重建，不查询当前 registry。 +- 声明缺失、无效或不可验证时，读取与重建都保持 `untrusted`/`unknown`。不做信任回填，既有写入权威、生成资格、 + 正文、content hash、citation 与搜索契约不变。 +- 第一阶段质量排序、生命周期操作与基于投递回执的使用反馈仍未实现。该能力只是其前置条件，不替代它们。 + ## 独立质量维度 四个维度必须独立： diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 5ab48cf7d6..c089367cba 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,6 +23,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -92,6 +93,7 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource + memory_evidence: MemoryEvidenceDeclaration | None = None def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index a1c4f174c8..0e5a90f9cd 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.5.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "892a9ad6b459026bb9626c755d7a0ef4eef3ed4688c507887a2feaab1736e525", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes." + "persistence_sha256": "1c3d8a5a79fe56abd1c1c2da9457b1f9dcb73f3cd38c0e82b4264afd99cba701", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 65914fe458..f5676ec3e0 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -32,6 +32,7 @@ ArtifactError, ArtifactFamilyMismatchError, ArtifactNotFoundError, + AtomicMemoryEvidenceError, ConnectorError, ConnectorSubmissionRejectedError, InvalidArtifactReferenceError, @@ -55,12 +56,14 @@ ) from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceAdapter, SourceCatalog, SourceCatalogBackend, SourceDefinition, SourceDefinitionRegistry, + SourceEvidence, SourceMaterialization, SourceProjection, SourceProjectionKey, @@ -82,6 +85,7 @@ "ArtifactRef", "ArtifactStore", "Artifacts", + "AtomicMemoryEvidenceError", "ConnectorError", "ConnectorSubmissionRejectedError", "InvalidArtifactReferenceError", @@ -94,6 +98,7 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", + "MemoryEvidenceDeclaration", "PolicyTransition", "PowerContext", "PowerContextError", @@ -108,6 +113,7 @@ "SourceDefinitionNotFoundError", "SourceDefinitionRegistry", "SourceError", + "SourceEvidence", "SourceMaterialization", "SourceNotFoundError", "SourceProjection", diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 33f3657012..f5c158d6e9 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -98,6 +98,9 @@ async def publish_many( records: Sequence[tuple[AtomicMemoryRecord, PreparedAtomicMemoryProjection]], ) -> None: ... async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: ... + async def capture_evidence( + self, connection: AsyncConnection, scope_id: str, record: AtomicMemoryRecord + ) -> None: ... ATOMIC_MERGE_ERRORS = ArtifactMergeErrors( @@ -316,7 +319,25 @@ def validate(self, *args, **kwargs): class _AtomicMemoryMergeService(ArtifactMergeService): - """Apply the same summary invariant to generic and Atomic-specific merge entry points.""" + """Persist descriptions and capture evidence atomically across all merge entry points.""" + + @override + async def _commit_locked( + self, + connection: AsyncConnection, + prepared: Sequence[tuple[PreparedArtifactMerge, SourceRef | None]], + current: dict[str, ArtifactMergeRecord], + context: Any, + source_read_set: _SourceReadSet | None, + ) -> tuple[ArtifactMergeMutationResult, ...]: + results = await super()._commit_locked(connection, prepared, current, context, source_read_set) + projections = cast(AtomicMemoryMergeAdapter, self.adapter).projections + for (item, _), result in zip(prepared, results, strict=True): + revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} + for record in result.records: + if record.ref.artifact_id in revisions: + await projections.capture_evidence(connection, item.plan.scope_id, _atomic_record(record)) + return results @override async def prepare_change(self, plan: ArtifactMergePlan) -> PreparedArtifactMerge: diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py new file mode 100644 index 0000000000..614db7b3cc --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -0,0 +1,266 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable per-revision evidence inputs and their neutral internal annotation.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Sequence +from typing import Literal, cast + +import rfc8785 +from aiosqlite import Connection as SQLiteConnection +from pydantic import BaseModel, ConfigDict, ValidationError +from sqlalchemy import insert, select +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE +from powercontext.builtin.persistence.sources import SourceRepository, _require_identity +from powercontext.errors import AtomicMemoryEvidenceError +from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources import SourceRef +from powercontext.sources.evidence import ( + MemoryEvidenceDeclaration, + SourceEvidence, + log_evidence_neutralized, +) + + +class AtomicMemoryEvidenceAnnotation(BaseModel): + """Exact revision facts, not a trust score or recursively inferred evidence.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + artifact_ref: ArtifactRef + sources: tuple[SourceEvidence, ...] + format: Literal["powercontext.atomic-memory.evidence.v1"] = "powercontext.atomic-memory.evidence.v1" + + +class AtomicMemoryEvidenceRepository: + """Join caller transactions; retained inputs are never deleted with the index. + + Accepted history is immutable. The index-owned projection can be discarded + and replaced from verified history without joining it into search. + """ + + async def save( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + ) -> AtomicMemoryEvidenceAnnotation: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is not None: + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + accepted = [] + for source in sources: + value = source.model_dump(mode="json") + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) + declaration = MemoryEvidenceDeclaration() + accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + annotation = AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(accepted) + ) + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + # SQLite legacy transaction mode does not BEGIN on SELECT; + # SAVEPOINT must not accidentally own and commit the write. + await connection.exec_driver_sql("BEGIN") + try: + async with connection.begin_nested(): + await connection.execute( + insert(ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE).values( + scope_id=scope_id, + artifact_id=artifact_ref.artifact_id, + revision=artifact_ref.revision, + sources=rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]), + ) + ) + except IntegrityError: + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is None: + raise + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + return annotation + + async def read( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + async def read_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + payload = await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + async def replace_projection(self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation) -> None: + reference = self._reference(annotation.scope_id, annotation.artifact_ref) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + payload = rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]) + if connection.dialect.name == "sqlite": + statement = sqlite_insert(table) + statement = statement.on_conflict_do_update( + index_elements=("scope_id", "artifact_id", "revision"), set_={"sources": statement.excluded.sources} + ) + else: + statement = mysql_insert(table) + statement = statement.on_duplicate_key_update(sources=statement.inserted.sources) + await connection.execute( + statement, + { + "scope_id": annotation.scope_id, + "artifact_id": reference.artifact_id, + "revision": reference.revision, + "sources": payload, + }, + ) + + async def verify( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + source_refs: Sequence[SourceRef], + *, + sources: SourceRepository | None = None, + ) -> AtomicMemoryEvidenceAnnotation: + """Derive exact facts from retained inputs, never today's declarations. + + Missing or damaged declaration records are neutral. Missing Sources, + corrupt authoritative payloads and unequal exact materials remain errors. + No accepted snapshot is saved or repaired by this operation. + """ + + from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY + + artifact_ref = self._reference(scope_id, artifact_ref) + repository = SourceRepository(BUILTIN_SOURCE_REGISTRY) if sources is None else sources + materials = await repository.historical_evidence(connection, scope_id, source_refs) + annotation = await self.read(connection, scope_id, artifact_ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryEvidenceError + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + log_evidence_neutralized("evidence_mismatch", unit="revision") + return self._neutral(scope_id, artifact_ref, materials) + verified = [] + for accepted, material in zip(annotation.sources, materials, strict=True): + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ): + verified.append(accepted) + continue + log_evidence_neutralized("evidence_mismatch", unit="source", level=logging.WARNING) + verified.append(accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()})) + return annotation.model_copy(update={"sources": tuple(verified)}) + + @staticmethod + def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + reference = ArtifactRef.model_validate(artifact_ref.model_dump()) + if reference.family != "atomic-memory": + from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError + + raise InvalidRepositoryArgumentError("artifact_ref", "must reference an Atomic Memory revision") + return reference + + @staticmethod + async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef) -> object | None: + table = ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE + return await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + + @staticmethod + def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: + try: + return _decode_annotation(scope_id, artifact_ref, payload) + except (ValueError, TypeError): + # An unreadable declaration snapshot conveys no historical claim. + # This does not catch SQL errors or actual Source resolution errors. + log_evidence_neutralized("decode_failed", unit="declaration", level=logging.WARNING) + return None + + @staticmethod + def _neutral( + scope_id: str, artifact_ref: ArtifactRef, sources: Sequence[SourceEvidence] + ) -> AtomicMemoryEvidenceAnnotation: + # Preserve a damaged row exactly, without replacing it with today's + # stronger configuration. A caller can only receive neutral facts. + return AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, + artifact_ref=artifact_ref, + sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), + ) + + +class _InvalidEvidenceSnapshot(ValueError): + """A retained evidence snapshot is not a decodable record list.""" + + +def _decode_annotation(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation: + """Decode one stored snapshot, raising when it is not a valid record list.""" + + if not isinstance(payload, (bytes, bytearray, memoryview)): + raise _InvalidEvidenceSnapshot + values = json.loads(bytes(payload)) + if not isinstance(values, list) or any(not isinstance(value, dict) for value in values): + raise _InvalidEvidenceSnapshot + sources = [] + for value in cast("list[dict[str, object]]", values): + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) + declaration = MemoryEvidenceDeclaration() + sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py new file mode 100644 index 0000000000..22846565c1 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retained exact-version evidence, independent of disposable search projections.""" + +from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, Table, Text +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT + +from powercontext.builtin.persistence.tables import SHARED_METADATA, identity_string +from powercontext.limits import ( + MAX_ARTIFACT_ID_LENGTH, + MAX_SCOPE_ID_LENGTH, + MAX_SOURCE_ID_LENGTH, + MAX_SOURCE_TYPE_LENGTH, +) + +ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION = "p0005" +ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE = Table( + "pc_atomic_memory_evidence_schema", + SHARED_METADATA, + Column("schema_version", identity_string(32), primary_key=True), +) +ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE = Table( + "pc_atomic_memory_evidence_history", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_history_revision"), +) +SOURCE_EVIDENCE_BINDINGS_TABLE = Table( + "pc_source_evidence_bindings", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), primary_key=True), + Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), primary_key=True), + Column("material_digest", identity_string(71), nullable=False), + Column("definition_name", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("definition_version", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("manifest_fingerprint", identity_string(71)), + Column("declaration", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), +) +ATOMIC_MEMORY_EVIDENCE_TABLES = ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index d4a7fe4ca8..e9d0e97b36 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -43,9 +43,19 @@ lexical_search_score, ) from powercontext.builtin.inference import EmbeddingModel -from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + ATOMIC_MEMORY_PROJECTION_FORMAT, +) +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import SourceEvidence AtomicMemorySearchMode = Literal["fts", "vector", "hybrid"] ATOMIC_MEMORY_RRF_CONSTANT = 60 @@ -190,6 +200,27 @@ async def replace_many( async def delete(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: ... + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: ... + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + + async def project_evidence( + self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation, / + ) -> None: ... + + async def read_evidence_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -215,11 +246,13 @@ def __init__( index: AtomicMemoryIndex, *, embedding_model: EmbeddingModel | None = None, + sources: SourceRepository | None = None, ) -> None: if not callable(getattr(index, "replace_many", None)): raise AtomicMemoryIndexError("publication", "Atomic Memory index must publish complete projection sets") self.index = index self._embedding_model = embedding_model + self._sources = sources @property def profile_fingerprint(self) -> str | None: @@ -270,6 +303,18 @@ async def prepare(self, content: BaseModel) -> PreparedAtomicMemoryProjection: embedding_input_hash=None if vector is None else atomic_memory_embedding_input_hash(kind, body), ) + async def capture_evidence(self, connection: AsyncConnection, scope_id: str, record: Any) -> None: + """Accept history only for a committed body write, never state-only republishing.""" + + artifact = record.artifact + if ( + self._sources is not None + and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None + ): + evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) + annotation = await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + await self.index.project_evidence(connection, annotation) + async def publish( self, connection: AsyncConnection, @@ -563,12 +608,42 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table,) + self.tables: tuple[Table, ...] = ( + table, + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + *ATOMIC_MEMORY_EVIDENCE_TABLES, + ) + self._evidence = AtomicMemoryEvidenceRepository() self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read(connection, scope_id, artifact_ref) + + async def project_evidence( + self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation, / + ) -> None: + await self._evidence.replace_projection(connection, annotation) + + async def read_evidence_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read_projection(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /) -> None: """Audit vector deployment readiness at startup, outside ordinary bounded search.""" profile = self.capabilities.embedding_profile diff --git a/src/powercontext/builtin/persistence/atomic_memory_index_schema.py b/src/powercontext/builtin/persistence/atomic_memory_index_schema.py index 848c572ba8..58cca6fc00 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index_schema.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index_schema.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The single rebuildable current Atomic Memory projection.""" +"""Rebuildable current and exact-version Atomic Memory index projections.""" from typing import Any from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, MetaData, String, Table, Text -from sqlalchemy.dialects.mysql import MEDIUMTEXT +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT from sqlalchemy.types import TypeEngine from powercontext.builtin.persistence.tables import identity_string @@ -26,6 +26,17 @@ ATOMIC_MEMORY_CURRENT_TABLE_NAME = "pc_atomic_memory_current" ATOMIC_MEMORY_PROJECTION_FORMAT = "powercontext.atomic-memory.current.v1" +# This table is disposable index state, never retained archive authority. +ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE = Table( + "pc_atomic_memory_evidence", + MetaData(), + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_revision"), +) + def atomic_memory_current_table(embedding_type: TypeEngine[Any] | None = None, /) -> Table: """Build one backend-owned table, with its deployment's native vector type. @@ -62,4 +73,9 @@ def atomic_memory_current_table(embedding_type: TypeEngine[Any] | None = None, / ) -__all__ = ["ATOMIC_MEMORY_CURRENT_TABLE_NAME", "ATOMIC_MEMORY_PROJECTION_FORMAT", "atomic_memory_current_table"] +__all__ = [ + "ATOMIC_MEMORY_CURRENT_TABLE_NAME", + "ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE", + "ATOMIC_MEMORY_PROJECTION_FORMAT", + "atomic_memory_current_table", +] diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index ff12a47d4b..641755fce8 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -25,7 +25,12 @@ from sqlalchemy import inspect, text from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( + AtomicMemoryEvidenceSchemaError, + require_atomic_memory_evidence_schema, +) LEGACY_CITATION_COLUMN = "memory_citations" _LEGACY_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions", "pc_candidate_versions") @@ -88,13 +93,28 @@ async def legacy_foreign_keys(connection: AsyncConnection, tables: set[str]) -> async def atomic_memory_readiness_issues( - connection: AsyncConnection, tables: set[str] | None = None, *, collections_removed: bool = True + connection: AsyncConnection, + tables: set[str] | None = None, + *, + collections_removed: bool = True, + require_evidence: bool = True, ) -> list[str]: - """Check public migration completion, allowing collections only during offline pre-removal acceptance.""" + """Check versioned schema and public migration completion without reading retained history. + + ``require_evidence`` only relaxes the check before the registered revision that + introduces the evidence tables; a migrated head is still verified by its schema. + """ if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] + if require_evidence and ( + "pc_artifacts" in tables or any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES) + ): + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables diff --git a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json index 9d6c8f65cc..d436535d18 100644 --- a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json +++ b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json @@ -101,12 +101,36 @@ "pc_atomic_memory_current_fts_idx", "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" ], + [ + "table", + "pc_atomic_memory_evidence", + "pc_atomic_memory_evidence", + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence_history", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_schema", + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)" + ], [ "table", "pc_memory_artifact_archive", "pc_memory_artifact_archive", "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tmetadata BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" ], + [ + "table", + "pc_source_evidence_bindings", + "pc_source_evidence_bindings", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" + ], [ "trigger", "pc_atomic_memory_fts_delete", @@ -129,6 +153,7 @@ "empty_tables": [ "pc_atomic_memory_current", "pc_atomic_memory_current_fts", + "pc_atomic_memory_evidence", "pc_memory_artifact_archive" ] } diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py new file mode 100644 index 0000000000..68c8d90087 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply the frozen additive p0005 revision to runtime-owned evidence tables. + +The production bundle covers a historical subset of business tables. Runtime +profiles therefore apply this same frozen additive revision independently of +that bundle's whole-database inventory, without stamping unrelated schema or +backfilling declarations for old Source/Artifact rows. SQLite DDL joins the +caller's transaction; MySQL DDL may commit and is resumed by exact postconditions. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from aiosqlite import Connection as SQLiteConnection +from sqlalchemy import inspect, select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION, + ATOMIC_MEMORY_EVIDENCE_TABLES, +) +from powercontext.builtin.persistence.errors import PersistenceError + + +class AtomicMemoryEvidenceSchemaError(PersistenceError): + """An evidence schema is not the registered versioned layout.""" + + def __init__(self, detail: str) -> None: + message = { + "missing": ( + "Atomic Memory evidence schema is incomplete; apply the registered evidence revision " + "(p0005 for runtime bundles, v1_5_0_002 in the Server chain)" + ), + "version": "Atomic Memory evidence schema version is unsupported", + "shape": "Atomic Memory evidence table has an incompatible shape", + }[detail] + super().__init__(message) + + +async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> None: + """Add only missing p0005 tables, validate exact shapes, then mark ready. + + No reads or writes of Source/Artifact history are needed, and no current + declaration is used to invent accepted historical inputs. + """ + + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + await connection.exec_driver_sql("BEGIN IMMEDIATE") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name in tables: + await _require_shape(connection, table.name, statement, dialect) + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + versions = ( + tuple((await connection.execute(select(marker.c.schema_version))).scalars()) if marker.name in tables else () + ) + if versions and versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + if versions and any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + # A ready marker cannot authorize recreating missing retained history. + raise AtomicMemoryEvidenceSchemaError("missing") + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name not in tables: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) + if not versions: + await connection.exec_driver_sql(statements[-1]) + + +async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tables: set[str] | None = None) -> None: + """Check retained evidence readiness without DDL or historical payload reads.""" + + if tables is None: + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + if any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + raise AtomicMemoryEvidenceSchemaError("missing") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + await _require_shape(connection, table.name, statement, dialect) + versions = tuple((await connection.execute(select(ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE.c.schema_version))).scalars()) + if versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + + +async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: + """Require the registered table shape: exact for SQLite, structural for MySQL. + + SQLite stores the executed CREATE text in ``sqlite_master``, so exact equality + is the check available without a second SQL parser; it also freezes the Server + bundle and the runtime revision to one byte-identical DDL copy, asserted by + ``tests/builtin/persistence/test_atomic_evidence_schema_copies.py``. MySQL + reuses the frozen inventory parser because dumps and replays may normalize + physical formatting. + """ + if dialect == "sqlite": + actual = await connection.exec_driver_sql( + "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " + "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", + (name, name), + ) + if actual.all() != [("table", name, statement)]: + raise AtomicMemoryEvidenceSchemaError("shape") + return + # Reuse the frozen MySQL inventory parser so physical formatting accepted + # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.models import MigrationError + from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table + + actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] + try: + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + except MigrationError as error: + raise AtomicMemoryEvidenceSchemaError("shape") from error + if not _matches( + {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, + {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, + ): + raise AtomicMemoryEvidenceSchemaError("shape") diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_references.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_references.py index c0b5b077df..2800a6f1a8 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_references.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_references.py @@ -1264,16 +1264,24 @@ async def remove_legacy_collections( async def residual_issues( # noqa: C901 - One flat list of independent residual checks. - connection: AsyncConnection, tables: set[str], *, thorough: bool, collections_removed: bool = True + connection: AsyncConnection, + tables: set[str], + *, + thorough: bool, + collections_removed: bool = True, + require_evidence: bool = True, ) -> list[str]: """Report public legacy objects; ``thorough`` also scans business payloads for typed legacy references. Before removal (``collections_removed=False``) the archived collections, their own lineage and the emptied legacy citation columns may still be - public; every other reference must be gone. + public; every other reference must be gone. ``require_evidence`` is false + only while a frozen pre-evidence migration is still converting history. """ - issues = await atomic_memory_readiness_issues(connection, tables, collections_removed=collections_removed) + issues = await atomic_memory_readiness_issues( + connection, tables, collections_removed=collections_removed, require_evidence=require_evidence + ) if not thorough: return issues diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py index 4e02275c41..9a36a1cf9c 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py @@ -1429,8 +1429,13 @@ async def apply_atomic_memory_migration( # noqa: C901 - One ordered offline mai snapshot_budget_bytes: int = DEFAULT_SNAPSHOT_BUDGET_BYTES, owner_decisions: Mapping[tuple[str, str, str], OwnerDecision] | None = None, require_owners: bool = False, + require_evidence: bool = True, ) -> AtomicMemoryMigrationReport: - """Archive, import, convert references, then remove legacy collections and citation columns from public tables.""" + """Archive, import, convert references, then remove legacy collections and citation columns from public tables. + + ``require_evidence`` is false only for a frozen pre-evidence revision that + converts history before the registered evidence tables exist. + """ if not maintenance_confirmed: raise AtomicMemoryMigrationError(("apply requires stopped old writers and --maintenance-confirmed",)) @@ -1552,6 +1557,7 @@ async def apply_atomic_memory_migration( # noqa: C901 - One ordered offline mai check_projection=not rebuilt_projection, thorough=True, collections_removed=False, + require_evidence=require_evidence, snapshot_budget_bytes=snapshot_budget_bytes, ) if not report.ready: @@ -1562,8 +1568,16 @@ async def apply_atomic_memory_migration( # noqa: C901 - One ordered offline mai raise AtomicMemoryMigrationError((str(error),)) from error async with database.transaction() as connection: await drop_legacy_citation_columns(connection) + if require_evidence and "pc_artifacts" in await _tables(connection): + from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_evidence + + # Imported revisions never accepted a declaration; derive neutral facts + # from the converted lineage without manufacturing historical snapshots. + await rebuild_atomic_memory_evidence(connection, index) async with database.transaction() as connection: - residual = await residual_issues(connection, await _tables(connection), thorough=False) + residual = await residual_issues( + connection, await _tables(connection), thorough=False, require_evidence=require_evidence + ) if rebuilt_projection: residual.append( "the obsolete development projection was recreated empty; run " @@ -1641,6 +1655,7 @@ async def _verify_atomic_memory_migration( # noqa: C901 - One frozen import ver check_projection: bool, thorough: bool, collections_removed: bool = True, + require_evidence: bool = True, snapshot_budget_bytes: int = DEFAULT_SNAPSHOT_BUDGET_BYTES, ) -> AtomicMemoryMigrationReport: @@ -1651,7 +1666,15 @@ async def _verify_atomic_memory_migration( # noqa: C901 - One frozen import ver return AtomicMemoryMigrationReport(action="verify", errors=error.errors) errors = list(inventory.errors) tables = await _tables(connection) - errors.extend(await residual_issues(connection, tables, thorough=thorough, collections_removed=collections_removed)) + errors.extend( + await residual_issues( + connection, + tables, + thorough=thorough, + collections_removed=collections_removed, + require_evidence=require_evidence, + ) + ) schema_errors = await _merge_schema_errors(connection, tables) required = {"pc_artifact_heads", "pc_artifact_lineage_artifacts"} if check_projection: diff --git a/src/powercontext/builtin/persistence/migrations/atomic_projection.py b/src/powercontext/builtin/persistence/migrations/atomic_projection.py index 13108742dd..c76ec7f8cc 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_projection.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_projection.py @@ -23,6 +23,11 @@ from sqlalchemy import Connection, select, tuple_ from typing_extensions import override +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex @@ -63,7 +68,11 @@ def _sqlite_helper_digest(connection, batch_size): class AtomicProjectionMigrationRunner(ManagedMaintenanceRunner): operation = "atomic_projection" - affected_objects: tuple[str, ...] = ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + affected_objects: tuple[str, ...] = ( + "pc_atomic_memory_current", + "pc_atomic_memory_current_fts", + "pc_atomic_memory_evidence", + ) def __init__(self, *args, **kwargs): super().__init__( @@ -75,9 +84,9 @@ def __init__(self, *args, **kwargs): **kwargs, ) self.affected_objects = ( - ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + ("pc_atomic_memory_current", "pc_atomic_memory_current_fts", "pc_atomic_memory_evidence") if isinstance(self.connections.config, SQLiteConfig) - else ("pc_atomic_memory_current",) + else ("pc_atomic_memory_current", "pc_atomic_memory_evidence") ) def _index(self, connection): @@ -87,10 +96,16 @@ def _index(self, connection): def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str, bool, bool]: check_server_layout(connection) index = self._index(connection) + # Built from the migration bundle, so no host SourceDefinitionRegistry exists: + # admission uses BUILTIN_SOURCE_REGISTRY while host reads use the configured one. try: report = finish( atomic_memory_rebuild.verify_atomic_memory_current( - pinned(connection), index, batch_size=self.batch_size + pinned(connection), + index, + batch_size=self.batch_size, + require_evidence=True, + require_evidence_projection=True, ) ) except Exception as error: @@ -122,6 +137,9 @@ def _rows_digest(self, connection, current): ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACT_LINEAGE_ARTIFACTS_TABLE, current, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, ): snapshot.update(table.name.encode()) keys = tuple(table.primary_key.columns) @@ -153,6 +171,7 @@ def _rows_digest(self, connection, current): @override def _apply_operation(self, connection: Connection, guard) -> None: index = self._index(connection) + # No host registry on this bundle path; see the note in _state. report = finish( atomic_memory_rebuild.rebuild_atomic_memory_current( pinned(connection), index, batch_size=self.batch_size, guard=guard diff --git a/src/powercontext/builtin/persistence/migrations/historical.py b/src/powercontext/builtin/persistence/migrations/historical.py index 1448093153..76ddb185af 100644 --- a/src/powercontext/builtin/persistence/migrations/historical.py +++ b/src/powercontext/builtin/persistence/migrations/historical.py @@ -32,6 +32,9 @@ from sqlalchemy.schema import CreateColumn from typing_extensions import override +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + from .bundle import MigrationBundle from .models import MigrationError, digest from .pinned import finish, pinned @@ -464,6 +467,15 @@ async def initialize(self, connection, /): await self.rebuild_fts(connection) class HistoricalNativeAtomicIndex(OceanBaseAtomicMemoryIndex): + def __init__(self, profile): + super().__init__(profile) + # Frozen historical DDL predates the registered evidence revisions. + # Creating these tables here would apply runtime DDL outside those + # revisions, leaving the Server chain's marker INSERT unsatisfiable. + deferred = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + deferred.add(ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE.name) + self.tables = tuple(table for table in self.tables if table.name not in deferred) + @override async def require_current_schema(self, connection): columns = await connection.run_sync(lambda c: inspect(c).get_columns("pc_atomic_memory_current")) @@ -485,7 +497,9 @@ async def require_current_schema(self, connection): raise MigrationError("embedding_required", "Memory conversion requires the configured embedding provider.") with _embedding(embedding_factory if profile is not None else None) as model: report = finish( - apply_atomic_memory_migration(database, index, maintenance_confirmed=True, embedding_model=model) + apply_atomic_memory_migration( + database, index, maintenance_confirmed=True, embedding_model=model, require_evidence=False + ) ) if not report.ready: raise MigrationError("data_conversion_required", "; ".join(report.errors)) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py new file mode 100644 index 0000000000..2550f917ed --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the frozen additive evidence layout without backfilling trust.""" + +from alembic import context, op + +revision = "p0005" +down_revision = "p0004" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) + op.execute("INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 7d1fec90bb..7a3d943bdb 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -37,6 +37,7 @@ _LEXEME = re.compile( r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+*/%\^&|~!?:@-]" ) + _CONTROL_TABLE = "pc_schema_revision" _PHYSICAL_VALUES = frozenset({ "row_format", diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 92f6ba7340..63887addc7 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -5,12 +5,16 @@ ], "p0002": [], "p0003": [], - "p0004": [] + "p0004": [], + "p0005": [ + "schemas/atomic_memory_evidence_v1.json" + ] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", - "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004" + "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004", + "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1": "p0005" }, "sqlite_fingerprints": { "p0001": [ @@ -26,6 +30,10 @@ "p0004": [ "f65244be963a382a62565118ff874cd775ab1ea33cf51a49881b40391e47c051", "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4" + ], + "p0005": [ + "10b8f62310f90e4e267e795efef0aa6e6716c1ecad515ab745e783507f6496f3", + "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1" ] }, "affected_objects": [ @@ -33,7 +41,10 @@ "pc_artifact_heads", "pc_artifact_candidate_versions", "pc_artifact_tags", - "pc_artifact_handles" + "pc_artifact_handles", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_source_evidence_bindings" ], "retained_objects": [ "pc_retained_p0002_artifact_tags" diff --git a/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json index 6d6bda0e94..6442266801 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json +++ b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json @@ -18,6 +18,11 @@ ], "p0004": [ "CREATE TABLE pc_artifact_handles (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\thandle_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\trevision INTEGER NOT NULL,\n\tPRIMARY KEY (scope_id, family, handle_key),\n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT,\n\tCONSTRAINT ck_pc_artifact_handles_revision_positive CHECK (revision > 0)\n)" + ], + "p0005": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" ] }, "checkpoints": { diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json new file mode 100644 index 0000000000..e4af1dc164 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json @@ -0,0 +1,14 @@ +{ + "sqlite": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ], + "mysql": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py new file mode 100644 index 0000000000..9190834ba0 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen additive evidence revision; never synthesize historical declarations.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0005" +down_revision = "p0004" + + +def upgrade() -> None: + statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text()) + connection = op.get_bind() + for statement in statements["sqlite" if connection.dialect.name == "sqlite" else "mysql"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py new file mode 100644 index 0000000000..66f5d7dfa5 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate retained evidence only after its introducing Server revision.""" + +from sqlalchemy import inspect, select + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import require_atomic_memory_evidence_schema +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.pinned import finish, pinned +from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex +from powercontext.builtin.runtime.atomic_memory_rebuild import verify_atomic_memory_current + + +def preflight(connection, *, configuration): + # MySQL may commit the marker table before the remaining DDL. Its row is + # inserted last; only that completed declaration requires full readiness. + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + if ( + marker.name in inspect(connection).get_table_names() + and connection.execute(select(marker.c.schema_version).limit(1)).first() is not None + ): + finish(require_atomic_memory_evidence_schema(pinned(connection))) + + +def validate(connection, *, configuration): + profile = ( + EmbeddingProfile.model_validate(configuration["embedding_profile"]) + if configuration.get("embedding_enabled") + else None + ) + index = ( + SQLiteAtomicMemoryIndex(profile) if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex(profile) + ) + report = finish(verify_atomic_memory_current(pinned(connection), index, require_evidence=True)) + if not report.ready: + raise MigrationError( + "projection_verification_failed", "Atomic Memory projections differ from their authoritative records." + ) + + +# This additive revision validates existing history; it never rewrites declarations. +apply = verify = validate diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py new file mode 100644 index 0000000000..cf6f0ce2e5 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rebuild disposable exact-revision evidence from retained declarations.""" + +from sqlalchemy import inspect, select + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import require_atomic_memory_evidence_schema +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.pinned import finish, pinned +from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex +from powercontext.builtin.runtime.atomic_memory_rebuild import ( + rebuild_atomic_memory_evidence, + verify_atomic_memory_evidence, +) + + +def _index(connection, configuration): + profile = ( + EmbeddingProfile.model_validate(configuration["embedding_profile"]) + if configuration.get("embedding_enabled") + else None + ) + return ( + SQLiteAtomicMemoryIndex(profile) if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex(profile) + ) + + +def preflight(connection, *, configuration): + # The marker table may precede other retained tables in a committed MySQL + # DDL prefix. Its final row distinguishes readiness from resumable work. + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + if ( + marker.name in inspect(connection).get_table_names() + and connection.execute(select(marker.c.schema_version).limit(1)).first() is not None + ): + finish(require_atomic_memory_evidence_schema(pinned(connection))) + + +def apply(connection, *, configuration): + finish(rebuild_atomic_memory_evidence(pinned(connection), _index(connection, configuration))) + validate(connection, configuration=configuration) + + +def validate(connection, *, configuration): + report = finish(verify_atomic_memory_evidence(pinned(connection), _index(connection, configuration))) + if not report.ready: + raise MigrationError( + "projection_verification_failed", "Atomic Memory evidence projections differ from retained declarations." + ) + + +verify = validate diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index c3b8d5de64..aadd20d129 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -29,13 +29,21 @@ "schemas/sqlite-v1_4_1_005.json", "schemas/mysql-v1_4_1_005.json" ], + "v1_4_1_006": [ + "schemas/sqlite-v1_4_1_006.json", + "schemas/mysql-v1_4_1_006.json" + ], "v1_5_0_001": [ "schemas/sqlite-v1_5_0_001.json", "schemas/mysql-v1_5_0_001.json" ], - "v1_4_1_006": [ - "schemas/sqlite-v1_4_1_006.json", - "schemas/mysql-v1_4_1_006.json" + "v1_5_0_002": [ + "schemas/sqlite-v1_5_0_002.json", + "schemas/mysql-v1_5_0_002.json" + ], + "v1_5_0_003": [ + "schemas/sqlite-v1_5_0_003.json", + "schemas/mysql-v1_5_0_003.json" ] }, "sqlite_baselines": { @@ -60,11 +68,17 @@ "v1_4_1_005": [ "2e24be9f699c202cb9d7985690b2be947bdc98f8f64c1c4e4b8e16e35a580fc4" ], + "v1_4_1_006": [ + "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" + ], "v1_5_0_001": [ "8f2d4a32f3558612b1bd390c262b8b3e355b172e34cc99692f38c11876226ebd" ], - "v1_4_1_006": [ - "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" + "v1_5_0_002": [ + "2aafcccabd74f8a8ddfe9f961b5ec3f01055110e7e961dfc27d4ec752e50806a" + ], + "v1_5_0_003": [ + "357287e3bae474ca1810dd0949fe56326dff48f5adf0db851b4b4cf3cf01a308" ] }, "affected_objects": [ @@ -122,6 +136,9 @@ "pc_atomic_memory_current_fts_data", "pc_atomic_memory_current_fts_docsize", "pc_atomic_memory_current_fts_idx", + "pc_atomic_memory_evidence", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence_schema", "pc_candidate_heads", "pc_candidate_versions", "pc_connector_checkpoints", @@ -155,7 +172,9 @@ "pc_skill_publications", "pc_source_cursors", "pc_source_definition_manifests", + "pc_source_evidence_bindings", "pc_source_journal_heads", + "pc_source_write_plugins", "pc_sources", "pc_topic_memory_active_chunks", "pc_topic_memory_active_topics", @@ -176,8 +195,7 @@ "pc_topic_memory_topic_fts_docsize", "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", - "pc_topic_memory_work_budgets", - "pc_source_write_plugins" + "pc_topic_memory_work_budgets" ], "readiness_scope": "server", "retained_objects": [ @@ -239,6 +257,34 @@ "seekdb", "oceanbase" ] + }, + { + "id": "atomic_memory_evidence", + "revision": "v1_5_0_002", + "kind": "projection", + "module": "atomic_memory_evidence.py", + "depends_on": [ + "portable_storage" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] + }, + { + "id": "atomic_memory_evidence_projection", + "revision": "v1_5_0_003", + "kind": "projection", + "module": "atomic_memory_evidence_projection.py", + "depends_on": [ + "atomic_memory_evidence" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -260,13 +306,23 @@ "server_upgrade", "portable_storage" ], + "v1_4_1_006": [ + "server_upgrade", + "portable_storage" + ], "v1_5_0_001": [ "artifact_storage", "portable_storage" ], - "v1_4_1_006": [ - "server_upgrade", - "portable_storage" + "v1_5_0_002": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence" + ], + "v1_5_0_003": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -284,8 +340,10 @@ "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", "schemas/mysql-v1_4_1_005.json", + "schemas/mysql-v1_4_1_006.json", "schemas/mysql-v1_5_0_001.json", - "schemas/mysql-v1_4_1_006.json" + "schemas/mysql-v1_5_0_002.json", + "schemas/mysql-v1_5_0_003.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py new file mode 100644 index 0000000000..912d439775 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register the retained evidence tables through the shared DDL executor.""" + +from alembic import context + +revision = "v1_5_0_002" +down_revision = "v1_5_0_001" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_003.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_003.py new file mode 100644 index 0000000000..b3897dfdf7 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_003.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Register the index-owned evidence projection through the shared DDL executor.""" + +from alembic import context + +revision = "v1_5_0_003" +down_revision = "v1_5_0_002" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json new file mode 100644 index 0000000000..839141a64a --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json @@ -0,0 +1,259 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "v1_5_0_002": { + "statements": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ], + "changes": [ + { + "pc_atomic_memory_evidence_schema": { + "columns": { + "schema_version": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "schema_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + } + }, + { + "pc_atomic_memory_evidence_history": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "sources": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_atomic_memory_evidence_history_revision": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + { + "pc_source_evidence_bindings": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "material_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "definition_name": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "definition_version": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "manifest_fingerprint": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "declaration": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "source_type", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + } + }, + {} + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_003.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_003.json new file mode 100644 index 0000000000..3fd312069e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_003.json @@ -0,0 +1,102 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "v1_5_0_003": { + "statements": [ + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ], + "changes": [ + { + "pc_atomic_memory_evidence": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "sources": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_atomic_memory_evidence_revision": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + } + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json new file mode 100644 index 0000000000..424938051e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json @@ -0,0 +1,8 @@ +{ + "create": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_003.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_003.json new file mode 100644 index 0000000000..9a726e95fa --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_003.json @@ -0,0 +1,5 @@ +{ + "create": [ + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence.py new file mode 100644 index 0000000000..ccbdad2f1b --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register retained Source evidence and exact Atomic Memory evidence snapshots.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "v1_5_0_002" +down_revision = "v1_5_0_001" + + +def upgrade(): + statements = json.loads((Path(__file__).parents[1] / "schemas/sqlite-v1_5_0_002.json").read_text()) + connection = op.get_bind() + for statement in statements["create"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_003_atomic_memory_evidence_projection.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_003_atomic_memory_evidence_projection.py new file mode 100644 index 0000000000..eb479d0f49 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_003_atomic_memory_evidence_projection.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Register disposable exact-revision Atomic Memory evidence projection.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "v1_5_0_003" +down_revision = "v1_5_0_002" + + +def upgrade(): + statements = json.loads((Path(__file__).parents[1] / "schemas/sqlite-v1_5_0_003.json").read_text()) + connection = op.get_bind() + for statement in statements["create"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json index 0940df10b5..4d8bd0a7bc 100644 --- a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json @@ -28,13 +28,21 @@ "schemas/sqlite-v1_4_1_005.json", "schemas/mysql-v1_4_1_005.json" ], + "v1_4_1_006": [ + "schemas/sqlite-v1_4_1_006.json", + "schemas/mysql-v1_4_1_006.json" + ], "v1_5_0_001": [ "schemas/sqlite-v1_5_0_001.json", "schemas/mysql-v1_5_0_001.json" ], - "v1_4_1_006": [ - "schemas/sqlite-v1_4_1_006.json", - "schemas/mysql-v1_4_1_006.json" + "v1_5_0_002": [ + "schemas/sqlite-v1_5_0_002.json", + "schemas/mysql-v1_5_0_002.json" + ], + "v1_5_0_003": [ + "schemas/sqlite-v1_5_0_003.json", + "schemas/mysql-v1_5_0_003.json" ] }, "sqlite_baselines": { @@ -59,11 +67,17 @@ "v1_4_1_005": [ "2e24be9f699c202cb9d7985690b2be947bdc98f8f64c1c4e4b8e16e35a580fc4" ], + "v1_4_1_006": [ + "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" + ], "v1_5_0_001": [ "8f2d4a32f3558612b1bd390c262b8b3e355b172e34cc99692f38c11876226ebd" ], - "v1_4_1_006": [ - "cacce80bdfb14aff889b7babebc9ab8487b7f0a11d2dbbd3d77a3717a23df74c" + "v1_5_0_002": [ + "2aafcccabd74f8a8ddfe9f961b5ec3f01055110e7e961dfc27d4ec752e50806a" + ], + "v1_5_0_003": [ + "357287e3bae474ca1810dd0949fe56326dff48f5adf0db851b4b4cf3cf01a308" ] }, "affected_objects": [ @@ -121,6 +135,9 @@ "pc_atomic_memory_current_fts_data", "pc_atomic_memory_current_fts_docsize", "pc_atomic_memory_current_fts_idx", + "pc_atomic_memory_evidence", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence_schema", "pc_candidate_heads", "pc_candidate_versions", "pc_connector_checkpoints", @@ -154,7 +171,9 @@ "pc_skill_publications", "pc_source_cursors", "pc_source_definition_manifests", + "pc_source_evidence_bindings", "pc_source_journal_heads", + "pc_source_write_plugins", "pc_sources", "pc_topic_memory_active_chunks", "pc_topic_memory_active_topics", @@ -175,8 +194,7 @@ "pc_topic_memory_topic_fts_docsize", "pc_topic_memory_topic_fts_idx", "pc_topic_memory_work_blocks", - "pc_topic_memory_work_budgets", - "pc_source_write_plugins" + "pc_topic_memory_work_budgets" ], "readiness_scope": "server", "retained_objects": [ @@ -243,6 +261,34 @@ "seekdb", "oceanbase" ] + }, + { + "id": "atomic_memory_evidence", + "revision": "v1_5_0_002", + "kind": "projection", + "module": "atomic_memory_evidence.py", + "depends_on": [ + "portable_storage" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] + }, + { + "id": "atomic_memory_evidence_projection", + "revision": "v1_5_0_003", + "kind": "projection", + "module": "atomic_memory_evidence_projection.py", + "depends_on": [ + "atomic_memory_evidence" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -264,13 +310,23 @@ "server_upgrade", "portable_storage" ], + "v1_4_1_006": [ + "server_upgrade", + "portable_storage" + ], "v1_5_0_001": [ "artifact_storage", "portable_storage" ], - "v1_4_1_006": [ - "server_upgrade", - "portable_storage" + "v1_5_0_002": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence" + ], + "v1_5_0_003": [ + "artifact_storage", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -288,8 +344,10 @@ "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", "schemas/mysql-v1_4_1_005.json", + "schemas/mysql-v1_4_1_006.json", "schemas/mysql-v1_5_0_001.json", - "schemas/mysql-v1_4_1_006.json" + "schemas/mysql-v1_5_0_002.json", + "schemas/mysql-v1_5_0_003.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index 96b7f736a6..c72fddc4b2 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -24,11 +24,18 @@ from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.schema import CreateTable, sort_tables +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.bootstrap import mysql_error_code +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: # noqa: C901 - Keep unmanaged backend creation and its derived structures together. - """Create selected tables only for unmanaged storage; managed schemas are read-only.""" + """Create selected tables only for unmanaged storage; managed schemas are read-only. + + Evidence tables are applied only through their registered additive migration on + every dialect, so a MySQL-family database never gets the tables without the + registered marker row. + """ from powercontext.builtin.persistence.tables import ARTIFACT_VECTORS_TABLE @@ -44,11 +51,17 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) "unsupported_configuration", "The versioned database does not cover the requested feature tables." ) return + evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): + await apply_atomic_memory_evidence_migration(connection) if connection.dialect.name == "mysql": - await connection.run_sync(lambda sync: _create_mysql_tables(sync, tables)) + selected = [table for table in tables if table.name not in evidence_names] + await connection.run_sync(lambda sync: _create_mysql_tables(sync, selected)) else: groups: dict[MetaData, list[Table]] = {} for table in tables: + if table.name in evidence_names: + continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 9e5d4a8640..fe0cddd470 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Persistence for worker-owned Source Definition manifests.""" +"""Immutable historical manifests owned by the Source registration boundary.""" from __future__ import annotations +import logging from collections.abc import Mapping from typing import Any @@ -26,15 +27,19 @@ from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, + InvalidStoredColumnError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinitionManifest +from powercontext.sources import SourceDefinition, SourceDefinitionManifest +from powercontext.sources.evidence import log_evidence_neutralized +from powercontext.sources.observations import _manifest_for_definition class SourceDefinitionManifestRepository: - """Register immutable worker-owned Definition manifests by name and version.""" + """Retain native and worker manifests without changing their identities.""" async def register( self, @@ -59,7 +64,9 @@ async def register( ) except IntegrityError: existing = await self.find(connection, manifest.name, manifest.version) - if existing is None or existing != manifest: + if existing is None: + raise + if existing != manifest: raise StoredPayloadConflictError( "source-definition-manifest", (manifest.name, manifest.version), @@ -67,6 +74,35 @@ async def register( return existing return manifest + async def register_native( + self, + connection: AsyncConnection, + definition: SourceDefinition[Any, Any, Any], + /, + ) -> SourceDefinitionManifest | None: + """Retain the existing manifest representation for trusted native material. + + This internal registration cannot replace an older manifest. A missing, + invalid or conflicting ancillary manifest leaves the Source declaration + neutral; database failures still abort the caller's transaction. External + registration keeps its existing active-Definition ownership guard. + """ + + try: + manifest = _manifest_for_definition(definition, transport=False) + except Exception: # Native schema providers may not expose a valid manifest. + log_evidence_neutralized("no_manifest", unit="manifest") + return None + try: + return await self.register(connection, manifest) + except StoredPayloadConflictError: + # Two manifests exist under one identity; evidence cannot choose either. + log_evidence_neutralized("manifest_conflict", unit="manifest", level=logging.WARNING) + return None + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) + return None + async def get( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 1488898501..baa01ff972 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -16,13 +16,20 @@ from __future__ import annotations +import logging from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress from dataclasses import dataclass +from hashlib import sha256 from typing import Any, Literal, cast import rfc8785 -from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from pydantic import AliasChoices, AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter +from referencing import Registry +from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncTransaction @@ -30,6 +37,7 @@ from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, @@ -40,16 +48,27 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import ( ARTIFACT_PROCESSING_BINDING_STATES_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, ) -from powercontext.builtin.sources.content import ContentSource +from powercontext.builtin.sources.content import CONTENT_SOURCE_NAME, ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import Source, SourceAdapter, SourceDefinitionRegistry, SourceObservation, SourceRef +from powercontext.sources import ( + MemoryEvidenceDeclaration, + Source, + SourceAdapter, + SourceDefinitionManifest, + SourceDefinitionRegistry, + SourceEvidence, + SourceObservation, + SourceRef, +) +from powercontext.sources.evidence import log_evidence_neutralized _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -60,6 +79,20 @@ PROFILE_SOURCE_WINDOW_BINDING, ) _SOURCE_LOOKUP_KEYS_PER_STATEMENT = 400 # Two bind values per key, including SQLite's 999-variable builds. +_VALIDATORS: dict[str, Any] = {} + + +def _source_validator(manifest: SourceDefinitionManifest) -> Any: + """Compile a retained schema once per immutable manifest fingerprint.""" + + validator = _VALIDATORS.get(manifest.fingerprint) + if validator is None: + schema = manifest.source_schema + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema, registry=Registry()) + if len(_VALIDATORS) < 128: + _VALIDATORS[manifest.fingerprint] = validator + return validator class _StoredSourcePayload(BaseModel): @@ -68,6 +101,9 @@ class _StoredSourcePayload(BaseModel): encoding: Literal["powercontext-source-v1"] representation: Literal["native", "observation"] value: dict[str, JsonValue] + # Server-owned header permits historical reads without adapter code. Never hash it. + source: Source | None = None + source_keys: dict[str, str] | None = None class StoredSource(BaseModel): @@ -153,6 +189,7 @@ async def add_many_with_status( existing = await self._existing_many(connection, scope_id, tuple(ref for _, ref, _ in prepared)) results: list[tuple[StoredSource, bool]] = [] new_rows = [] + new_sources: dict[tuple[str, str], tuple[SourceRef, Source]] = {} upgrades: dict[tuple[str, str], bytes] = {} for source, ref, payload in prepared: key = (ref.source_type, ref.source_id) @@ -170,11 +207,14 @@ async def add_many_with_status( upgrades[key] = payload elif stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) + else: + await self._retain_native_header(connection, scope_id, ref, source) results.append((stored, False)) continue position += 1 stored = StoredSource(ref=ref, value=source, journal_position=position) existing[key] = stored + new_sources[key] = (ref, source) new_rows.append({ "scope_id": scope_id, "source_type": ref.source_type, @@ -191,6 +231,11 @@ async def add_many_with_status( .values(position=position) ) await connection.execute(insert(SOURCES_TABLE), new_rows) + # Reuse only within this journal-locked batch; a rollback or later + # registration must never inherit an earlier transaction's result. + native_manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} + for ref, source in new_sources.values(): + await self._bind_memory_evidence(connection, scope_id, ref, source, native_manifests) for (source_type, source_id), payload in upgrades.items(): await connection.execute( update(SOURCES_TABLE) @@ -307,6 +352,440 @@ async def preload( connection, connection.get_transaction(), connection.get_nested_transaction(), scope_id, rows ) + async def evidence( + self, + connection: AsyncConnection, + scope_id: str, + refs: Sequence[SourceRef], + /, + ) -> tuple[SourceEvidence, ...]: + """Read accepted declarations against exact stored Source material. + + Source reads retain their existing errors and never become generation + eligibility checks. Missing or invalid declaration bindings are neutral; + database failures propagate. No current registry declaration or provider + value is substituted for historical registration facts. + """ + + return await self.historical_evidence(connection, scope_id, refs) + + async def historical_evidence( + self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / + ) -> tuple[SourceEvidence, ...]: + """Check retained material without loading today's native Definition. + + Only the stored representation selects decoding. Canonical JSON retains + all native fields, rather than round-tripping through a newer model. + This does not supply a decoder or provider value for ordinary reads. + """ + + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) + if not identities: + return () + by_ref: dict[tuple[str, str], Mapping[Any, Any]] = {} + for offset in range(0, len(identities), _SOURCE_LOOKUP_KEYS_PER_STATEMENT): + rows = ( + ( + await connection.execute( + select(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == scope_id, + tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_( + identities[offset : offset + _SOURCE_LOOKUP_KEYS_PER_STATEMENT] + ), + ) + ) + ) + .mappings() + .all() + ) + by_ref.update(((row["source_type"], row["source_id"]), row) for row in rows) + materials: list[SourceEvidence] = [] + fingerprints: dict[tuple[str, str], str | None] = {} + for ref in identities: + reference = SourceRef(source_type=ref[0], source_id=ref[1]) + row = by_ref.get(ref) + if row is None: + raise RepositoryNotFoundError("source", (scope_id, reference)) + source, material, verifiable = await self._historical_material(connection, reference, row) + if not verifiable: + fingerprints[ref] = None # Deliberately unverifiable, not a fingerprint mismatch. + if isinstance(source, SourceObservation): + fingerprints[ref] = source.definition_fingerprint + elif ref[0] == CONTENT_SOURCE_NAME: + # The server-owned Content witness is the sole permitted material + # mutation; this exclusion is the same one used at acceptance. + internal = material.get("internal") + if ( + isinstance(internal, dict) + and internal.get("role") == "lineage_only" + and internal.get("operation") in {"artifact_create", "artifact_replace"} + ): + continue + material = {key: value for key, value in material.items() if key != "handoff_receipt"} + materials.append( + SourceEvidence( + source_ref=reference, + materialization=source.materialization, + material_digest=_source_material_digest_material(material), + definition_name=ref[0], + definition_version=source.definition_version, + ) + ) + return await self._bound_evidence(connection, scope_id, tuple(materials), fingerprints) + + async def _historical_material( + self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Validate the stored representation and its retained schema, not a live model.""" + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) + except InvalidStoredPayloadError: + return await self._historical_legacy_material(connection, reference, row, payload) + material = envelope.value + source = ( + await self._historical_canonical_source(connection, reference, row, material) + if envelope.representation == "native" and envelope.source is None + else None + ) + if source is None: + source = self._historical_source(reference, envelope) + decoded = SourceRef( + source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, + source_id=source.name, + ) + if reference != decoded: + raise IdentityMismatchError("source", reference, decoded) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.manifest_fingerprint, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) + ) + ).one_or_none() + manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) + if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: + try: + value = source.payload if isinstance(source, SourceObservation) else material + _source_validator(manifest).validate(value) + except JsonSchemaValidationError as error: + if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( + material + ): + # Older releases bound declarations without checking serialization + # compatibility. Exact accepted bytes remain readable, but neutral. + log_evidence_neutralized("schema_mismatch", unit="source", level=logging.DEBUG) + return source, material, False + raise InvalidStoredPayloadError( + "source", reference.source_type, "payload does not match the historical schema" + ) from error + except (SchemaError, Unresolvable): + self._validate_unbound_material(reference, envelope, source) + else: + self._validate_unbound_material(reference, envelope, source) + return source, material, True + + async def _historical_legacy_material( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + payload: bytes, + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Read bare legacy material without guessing absent identity fields.""" + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + source = await self._historical_canonical_source(connection, reference, row, material) + if source is not None: + return source, material, False + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if not {"name", "definition_version", "materialization"} <= source.model_fields_set: + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) from None + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + else: + source = self.decode_source(row).value + return source, material, True + + async def _historical_canonical_source( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + material: dict[str, JsonValue], + ) -> Source | None: + """Recover evidence identity from canonical retained material, never a live alias.""" + + if not {"name", "definition_version", "materialization"} <= material.keys(): + return None + if material["name"] != reference.source_id: + return None + digest_material = material + if reference.source_type == CONTENT_SOURCE_NAME: + digest_material = {key: value for key, value in material.items() if key != "handoff_receipt"} + material_digest = _source_material_digest_material(digest_material) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.definition_name, table.c.definition_version, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + ) + ) + ).one_or_none() + if binding is not None: + if ( + binding.definition_name != reference.source_type + or binding.material_digest != material_digest + or binding.definition_version != material["definition_version"] + ): + return None + version = binding.definition_version + elif isinstance(material["definition_version"], str): + # Canonical retained identity is explicit even when its historical + # evidence binding predates the evidence tables. It stays neutral. + version = material["definition_version"] + else: + return None + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + definition = None + if ( + definition is not None + and definition.version == version + and any( + ( + ( + definition.source_class.model_fields[key].alias is not None + and definition.source_class.model_fields[key].alias != key + ) + or ( + definition.source_class.model_fields[key].serialization_alias is not None + and definition.source_class.model_fields[key].serialization_alias != key + ) + or ( + definition.source_class.model_fields[key].validation_alias is not None + and definition.source_class.model_fields[key].validation_alias != key + ) + ) + for key in ("definition_version", "materialization") + ) + ): + # The current exact-version codec is required to disambiguate retained identity keys. + return None + try: + return Source.model_validate({ + "name": reference.source_id, + "definition_version": version, + "materialization": material["materialization"], + "description": material.get("description"), + }) + except (TypeError, ValueError): + return None + + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: + """Decode identity independently of retained material and today's business defaults.""" + + material = envelope.value + if envelope.representation == "observation": + return load_model(SourceObservation, rfc8785.dumps(material), kind="source", name=reference.source_type) + if envelope.source is not None: + keys = envelope.source_keys or {key: key for key in ("name", "definition_version", "materialization")} + for key, alias in keys.items(): + value = getattr(envelope.source, key) + if alias not in material or material[alias] != value: + raise IdentityMismatchError("source-envelope", value, material.get(alias)) + return envelope.source + # Legacy aliased rows need their registered decoder once; canonical rows do not. + if {"name", "definition_version", "materialization"} <= material.keys() and material[ + "name" + ] == reference.source_id: + definition = None + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if definition is None or not _claims_identity_keys(definition.source_class): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + header = material + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + header = _normalize_native_material(definition.source_class, material) + if not {"name", "definition_version", "materialization"} <= header.keys(): + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) + try: + return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) + except InvalidStoredPayloadError: + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: + """Keep supported exact-read errors when no historical schema is available. + + A live decoder validates only material of its own version; it never + contributes a declaration or reinterprets a valid historical binding. + """ + + if envelope.representation == "observation": + return # SourceObservation already validated its envelope and payload. + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + # No configured Definition can perform the live-decoder cross-check. + log_evidence_neutralized("definition_missing", unit="source", level=logging.DEBUG) + return + if _claims_identity_keys(definition.source_class): + # The retained canonical identity/header is authoritative; a live business alias may claim it. + return + if envelope.source is None and any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + # Today's identity aliases cannot validate an older unbound representation. + return + if definition.version == source.definition_version: + self._decode_envelope(reference.source_type, envelope) + + @staticmethod + async def _bound_evidence( + connection: AsyncConnection, + scope_id: str, + materials: tuple[SourceEvidence, ...], + fingerprints: Mapping[tuple[str, str], str | None], + ) -> tuple[SourceEvidence, ...]: + if not materials: + return () + table = SOURCE_EVIDENCE_BINDINGS_TABLE + identities = tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) + bindings: dict[tuple[str, str], Mapping[Any, Any]] = {} + for offset in range(0, len(identities), _SOURCE_LOOKUP_KEYS_PER_STATEMENT): + rows = ( + ( + await connection.execute( + select(table).where( + table.c.scope_id == scope_id, + tuple_(table.c.source_type, table.c.source_id).in_( + identities[offset : offset + _SOURCE_LOOKUP_KEYS_PER_STATEMENT] + ), + ) + ) + ) + .mappings() + .all() + ) + bindings.update(((row["source_type"], row["source_id"]), row) for row in rows) + manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} + evidence: list[SourceEvidence] = [] + for value in materials: + identity = (value.source_ref.source_type, value.source_ref.source_id) + if identity in fingerprints and fingerprints[identity] is None: + evidence.append(value) + continue + binding = bindings.get(identity) + if binding is None: + # Captured before evidence bindings existed; no declaration was accepted. + log_evidence_neutralized("legacy_unverifiable", unit="source", level=logging.DEBUG) + evidence.append(value) + continue + if ( + binding["material_digest"] != value.material_digest + or binding["definition_name"] != value.definition_name + or binding["definition_version"] != value.definition_version + or ( + binding["manifest_fingerprint"] is not None and not isinstance(binding["manifest_fingerprint"], str) + ) + ): + log_evidence_neutralized("digest_mismatch", unit="source", level=logging.WARNING) + evidence.append(value) + continue + if binding["manifest_fingerprint"] is None: + log_evidence_neutralized("no_manifest", unit="manifest") + evidence.append(value) + continue + definition_identity = (value.definition_name, value.definition_version) + if definition_identity not in manifests: + manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) + manifest = manifests[definition_identity] + if manifest is None: + log_evidence_neutralized("no_manifest", unit="manifest") + evidence.append(value) + continue + evidence.append( + _apply_bound_declaration(value, binding, manifest, fingerprints.get(identity, manifest.fingerprint)) + ) + return tuple(evidence) + + async def _bind_memory_evidence( + self, + connection: AsyncConnection, + scope_id: str, + ref: SourceRef, + source: Source, + native_manifests: dict[tuple[str, str], SourceDefinitionManifest | None], + ) -> None: + """Freeze registration evidence only for a newly accepted Source row.""" + + if isinstance(source, SourceObservation): + manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint != source.definition_fingerprint: + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) + manifest = None + else: + definition = self._registry.definition_for_source(source) + identity = (definition.name, definition.version) + if identity not in native_manifests: + native_manifests[identity] = await SourceDefinitionManifestRepository().register_native( + connection, definition + ) + manifest = native_manifests[identity] + if manifest is not None: + # A native decoder can accept representations its generated schema + # cannot describe. Such captures are valid, but cannot attest evidence. + try: + _source_validator(manifest).validate(source.model_dump(mode="json", by_alias=True)) + except (JsonSchemaValidationError, SchemaError, Unresolvable): + log_evidence_neutralized("schema_mismatch", unit="source", level=logging.DEBUG) + manifest = None + declaration = ( + MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) + ) + await connection.execute( + insert(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + scope_id=scope_id, + source_type=ref.source_type, + source_id=ref.source_id, + material_digest=_source_material_digest(source), + definition_name=ref.source_type, + definition_version=source.definition_version, + manifest_fingerprint=None if manifest is None else manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + async def list( self, connection: AsyncConnection, @@ -473,6 +952,47 @@ def decode_source(self, row: Mapping[Any, Any], /) -> StoredSource: journal_position=int(row["journal_position"]), ) + async def _retain_native_header( + self, connection: AsyncConnection, scope_id: str, ref: SourceRef, source: Source + ) -> None: + """Enrich only an identical replay's unreconstructible legacy header. + + The retained value, binding, manifest, digest and journal position never change. + This grants no authority and does not publish another processing intent. + """ + + if isinstance(source, SourceObservation): + return + row = await self._find_row(connection, scope_id, ref) + if row is None: + # Same-batch duplicates have no retained row until the bulk insert. + return + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=ref.source_type) + except InvalidStoredPayloadError: + envelope = _StoredSourcePayload( + encoding="powercontext-source-v1", + representation="native", + value=_JSON_OBJECT.validate_json(payload, strict=True), + ) + if envelope.source is not None: + return + header = _source_header(source) + with suppress(InvalidStoredPayloadError): + if load_model(Source, rfc8785.dumps(envelope.value), kind="source", name=ref.source_type) == header: + return + enriched = envelope.model_copy(update={"source": header, "source_keys": _source_keys(source, envelope.value)}) + await connection.execute( + update(SOURCES_TABLE) + .where( + SOURCES_TABLE.c.scope_id == scope_id, + SOURCES_TABLE.c.source_type == ref.source_type, + SOURCES_TABLE.c.source_id == ref.source_id, + ) + .values(payload=rfc8785.dumps(enriched.model_dump(mode="json", exclude_none=True))) + ) + def _decode_envelope( self, source_type: str, @@ -491,11 +1011,15 @@ def _decode_envelope( definition = self._definition_by_name(source_type) source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, envelope.value)), kind="source", name=source_type, ) self._registry.definition_for_source(source) + if envelope.source is not None: + for key in ("name", "definition_version", "materialization"): + if getattr(envelope.source, key) != getattr(source, key): + raise IdentityMismatchError("source-envelope", envelope.source, _source_header(source)) return source, SourceRef(source_type=definition.name, source_id=source.name) def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Source, SourceRef]: @@ -510,9 +1034,13 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour ) decoded = SourceRef(source_type=source.source_type, source_id=source.name) else: + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", source_type, "payload is not JSON") from error source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, material)), kind="source", name=source_type, ) @@ -521,6 +1049,139 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded +def _apply_bound_declaration( + value: SourceEvidence, + binding: Mapping[Any, Any], + manifest: SourceDefinitionManifest, + expected_fingerprint: str | None, +) -> SourceEvidence: + """Apply only a declaration bound to the verified historical manifest.""" + + if manifest.fingerprint != binding["manifest_fingerprint"] or expected_fingerprint != manifest.fingerprint: + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) + return value + try: + declaration = MemoryEvidenceDeclaration.model_validate_json( + stored_bytes(binding["declaration"], column="declaration") + ) + except (InvalidStoredColumnError, TypeError, ValueError): + log_evidence_neutralized("decode_failed", unit="declaration", level=logging.WARNING) + declaration = MemoryEvidenceDeclaration() + return value.model_copy( + update={ + "manifest_fingerprint": manifest.fingerprint, + "declaration": declaration, + } + ) + + +def _source_material_digest(source: Source) -> str: + # Preserve every material field except the narrow server witness upgrade. + material = source.model_dump(mode="json", by_alias=True) + if isinstance(source, ContentSource): + material.pop("handoff_receipt", None) + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _management_write_source(source: Source) -> bool: + return ( + isinstance(source, ContentSource) + and source.internal is not None + and source.internal.role == "lineage_only" + and source.internal.operation in {"artifact_create", "artifact_replace"} + ) + + +async def _find_evidence_manifest( + connection: AsyncConnection, name: str, version: str +) -> SourceDefinitionManifest | None: + try: + return await SourceDefinitionManifestRepository().find(connection, name, version) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + return None + + +def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _source_header(source: Source) -> Source: + return Source(**{key: getattr(source, key) for key in Source.model_fields}) + + +def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: + """Retain the exact serialized spelling used for each native identity.""" + keys: dict[str, str] = {} + for key in ("name", "definition_version", "materialization"): + field = type(source).model_fields[key] + preferred = field.serialization_alias or key + candidates = [preferred] + if isinstance(field.validation_alias, str): + candidates.append(field.validation_alias) + elif isinstance(field.validation_alias, AliasChoices): + candidates.extend(choice for choice in field.validation_alias.choices if isinstance(choice, str)) + if field.alias is not None: + candidates.append(field.alias) + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is not None: + candidates.append(generator(key)) + if material is not None: + keys[key] = next( + (alias for alias in dict.fromkeys(candidates) if material.get(alias) == getattr(source, key)), + preferred, + ) + else: + keys[key] = preferred + return keys + + +def _claims_identity_keys(source_class: type[Source]) -> bool: + """Report whether a business field serializes to a canonical identity spelling. + + A headerless envelope cannot distinguish an identity value from a business + value with that name. When the registered model claims the spelling for a + different field, only its declared aliases are authoritative. + """ + + return any( + name not in Source.model_fields and key in (field.alias, field.validation_alias, field.serialization_alias) + for name, field in source_class.model_fields.items() + for key in Source.model_fields + ) + + +def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: + """Read pre-canonical inherited envelope aliases, never business-field aliases. + + Derive spellings from the registered generator, not a version/key allowlist. + Explicit adapter aliases and conflicting identities retain strict validation. + """ + + generator = source_class.model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is None: + return material + normalized = dict(material) + for key in Source.model_fields: + field = source_class.model_fields[key] + if field.validation_alias != key and field.alias_priority != 1: + continue + alias = generator(key) + if any( + field_name != key and alias in (field.alias, field.validation_alias, field.serialization_alias) + for field_name, field in source_class.model_fields.items() + ): + continue + if alias != key and alias in normalized: + if key in normalized and normalized[key] != normalized[alias]: + raise IdentityMismatchError("source", normalized[key], normalized[alias]) + normalized[key] = normalized.pop(alias) + return normalized + + def _dump_source( source: Source, *, @@ -532,8 +1193,16 @@ def _dump_source( encoding="powercontext-source-v1", representation=representation, value=_JSON_OBJECT.validate_python(source.model_dump(mode="json", by_alias=True)), + source=None if representation == "observation" else _source_header(source), + source_keys=None if representation == "observation" else _source_keys(source), ) - return rfc8785.dumps(envelope.model_dump(mode="json")) + if envelope.source_keys is not None: + for key, alias in envelope.source_keys.items(): + if alias not in envelope.value: + raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") + if envelope.value[alias] != getattr(source, key): + raise InvalidStoredPayloadError("source", name, f"serialized native identity changes {key!r}") + return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 853da1b1a9..bdda979a6e 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -76,6 +76,10 @@ verdict_key, ) from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) from powercontext.builtin.persistence.candidates import decode_candidate_proposal_envelope from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( @@ -118,6 +122,7 @@ from powercontext.builtin.tags import normalize_tags from powercontext.errors import SourceError from powercontext.sources import SourceDefinitionManifest, SourceObservation +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence FORMAT_VERSION = 1 _RECORDS_NAME = "records.ndjson" @@ -169,6 +174,7 @@ "source_definition", "source_journal_head", "source", + "source_evidence_binding", "skill_package", "artifact_revision", "artifact_handle", @@ -178,6 +184,7 @@ "artifact_head", "memory_entry_version", "memory_entry_head", + "atomic_memory_evidence_history", "candidate_version", "candidate_head", "topic_memory_publication", @@ -312,6 +319,11 @@ class _RecordSpec: ("scope_id", "source_type", "source_id"), ("payload", "journal_position"), ), + "source_evidence_binding": _RecordSpec( + SOURCE_EVIDENCE_BINDINGS_TABLE, + ("scope_id", "source_type", "source_id"), + ("material_digest", "definition_name", "definition_version", "manifest_fingerprint", "declaration"), + ), "skill_package": _RecordSpec( SKILL_PACKAGES_TABLE, ("scope_id", "tree_digest"), @@ -386,6 +398,11 @@ class _RecordSpec: ("scope_id", "memory_artifact_id", "entry_id"), ("family", "head_revision", "entry_version_id", "entry_content_hash"), ), + "atomic_memory_evidence_history": _RecordSpec( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + ("scope_id", "artifact_id", "revision"), + ("sources",), + ), "candidate_version": _RecordSpec( CANDIDATE_VERSIONS_TABLE, ("scope_id", "candidate_id", "version"), @@ -1020,14 +1037,9 @@ async def _export_rows( return # Definition manifests are global, but export authority is scope-bound. - # Include only immutable declarations referenced by selected observations. - definitions: set[tuple[str, str]] = set() - async for row in _export_rows(connection, "source", scopes): - observation = _source_observation(_record_from_row("source", row)) - if observation is not None: - definitions.add((observation.source_type, observation.definition_version)) + # Include immutable manifests referenced by observations or retained bindings. table = SOURCE_DEFINITION_MANIFESTS_TABLE - for name, definition_version in sorted(definitions): + for name, definition_version in sorted(await _referenced_definitions(connection, scopes)): row = ( ( await connection.execute( @@ -1040,10 +1052,31 @@ async def _export_rows( .one_or_none() ) if row is None: - raise BundleFormatError("observation references missing Source definition") + raise BundleFormatError("evidence or observation references missing Source definition") yield cast(Mapping[str, Any], row) +async def _referenced_definitions(connection: AsyncConnection, scopes: tuple[str, ...]) -> set[tuple[str, str]]: + """Collect Definition identities whose immutable manifest must travel with the scope.""" + + definitions: set[tuple[str, str]] = set() + async for row in _export_rows(connection, "source", scopes): + observation = _source_observation(_record_from_row("source", row)) + if observation is not None: + definitions.add((observation.source_type, observation.definition_version)) + async for binding in _export_rows(connection, "source_evidence_binding", scopes): + if binding["manifest_fingerprint"] is not None: + definitions.add((str(binding["definition_name"]), str(binding["definition_version"]))) + async for row in _export_rows(connection, "atomic_memory_evidence_history", scopes): + record = _record_from_row("atomic_memory_evidence_history", row) + definitions.update( + (source.definition_name, source.definition_version) + for source in _evidence_history_value(record) + if source.manifest_fingerprint is not None + ) + return definitions + + def _source_observation(record: _Record) -> SourceObservation | None: try: value = json.loads(cast(bytes, _database_value(record.payload["payload"]))) @@ -1488,6 +1521,30 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: strings("fingerprint") bytes_values("manifest") _source_definition(record) + elif kind == "source_evidence_binding": + strings("material_digest", "definition_name", "definition_version") + strings("manifest_fingerprint", nullable=("manifest_fingerprint",)) + bytes_values("declaration") + try: + MemoryEvidenceDeclaration.model_validate_json(cast(bytes, _database_value(payload["declaration"]))) + except (ValidationError, ValueError, TypeError) as error: + raise BundleFormatError("Source evidence binding has an invalid declaration") from error + if ( + payload["definition_name"] != identity["source_type"] + or not _valid_digest(payload["material_digest"]) + or (payload["manifest_fingerprint"] is not None and not _valid_digest(payload["manifest_fingerprint"])) + ): + raise BundleFormatError("Source evidence binding identity or digest is invalid") + elif kind == "atomic_memory_evidence_history": + bytes_values("sources") + sources = _evidence_history_value(record) + if any( + not _valid_digest(source.material_digest) + or source.definition_name != source.source_ref.source_type + or (source.manifest_fingerprint is not None and not _valid_digest(source.manifest_fingerprint)) + for source in sources + ): + raise BundleFormatError("Atomic Memory evidence has invalid bindings") elif kind == "source": bytes_values("payload") integers("journal_position", positive=True) @@ -1699,6 +1756,19 @@ def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObserva return value +def _evidence_history_value(record: _Record, /) -> tuple[SourceEvidence, ...]: + try: + value = json.loads(cast(bytes, _database_value(record.payload["sources"]))) + except (ValueError, UnicodeDecodeError, TypeError) as error: + raise BundleFormatError("Atomic Memory evidence payload is not valid JSON") from error + if not isinstance(value, list): + raise BundleFormatError("Atomic Memory evidence payload is not a list") + try: + return tuple(SourceEvidence.model_validate(item) for item in value) + except ValidationError as error: + raise BundleFormatError("Atomic Memory evidence payload has an invalid Source entry") from error + + def _validate_datetime_text(value: object, /) -> None: try: parsed = datetime.fromisoformat(cast(str, value)) @@ -2031,6 +2101,46 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis }, ): raise BundleFormatError("observation references missing or mismatched Source definition") + elif record.record_type == "source_evidence_binding": + if not contains( + "source", + {"scope_id": scope, "source_type": identity["source_type"], "source_id": identity["source_id"]}, + ): + raise BundleFormatError("Source evidence binding references missing Source") + fingerprint = payload["manifest_fingerprint"] + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": payload["definition_name"], + "definition_version": payload["definition_version"], + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Source evidence binding references missing Source definition") + elif record.record_type == "atomic_memory_evidence_history": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": "atomic-memory", + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing exact Artifact revision") + for source in _evidence_history_value(record): + if not contains("source", {"scope_id": scope, **source.source_ref.model_dump()}): + raise BundleFormatError("Atomic Memory evidence references missing Source") + fingerprint = source.manifest_fingerprint + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": source.definition_name, + "definition_version": source.definition_version, + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing Source definition") elif record.record_type in {"recurrence_match", "recurrence_observation"}: value = _recurrence_value(record) if not contains( diff --git a/src/powercontext/builtin/runtime/__init__.py b/src/powercontext/builtin/runtime/__init__.py index 5f35fbd0b6..8618be4519 100644 --- a/src/powercontext/builtin/runtime/__init__.py +++ b/src/powercontext/builtin/runtime/__init__.py @@ -42,6 +42,7 @@ GetDreamRunRequest, ListDreamRunsRequest, ) +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation from powercontext.builtin.review.generation import ( GeneratedCandidateResult, GenerationCapabilityUnavailableError, @@ -266,6 +267,7 @@ "ArtifactProcessingWorkerCompletion", "ArtifactProcessingWorkerFailure", "ArtifactProcessingWorkerOutcome", + "AtomicMemoryEvidenceAnnotation", "AtomicMemoryPage", "AtomicMemoryRuntimeApplication", "AtomicMemorySearchHit", diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index ec4cf0528d..dd31c8272d 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -3755,6 +3755,12 @@ async def state(self, artifact_id: str, *, context=None): async with self._runtime._scope_operation(self.scope_id): return await self._scoped.state(artifact_id, context=context) + async def evidence(self, artifact_id: str, *, revision: int, context=None): + """Read one authorized revision's neutral evidence facts.""" + + async with self._runtime._scope_operation(self.scope_id): + return await self._scoped.evidence(artifact_id, revision=revision, context=context) + async def read_dependency_snapshot( self, roots: tuple[ArtifactRef, ...], diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index 436bcaff68..5f71f41fd6 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -80,6 +80,10 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -327,6 +331,7 @@ def __init__( self.publisher = AtomicMemoryProjectionPublisher( index, embedding_model=embedding_model, + sources=artifacts.source_repository, ) self.service = AtomicMemoryService( artifacts=artifacts, @@ -377,6 +382,23 @@ async def get(self, artifact_id: str, *, revision: int | None = None, context=No connection, self.scope_id, artifact_id, context, revision=revision ) + async def evidence(self, artifact_id: str, *, revision: int, context=None) -> AtomicMemoryEvidenceAnnotation: + """Inspect neutral evidence facts for one authorized exact body revision. + + A missing historical snapshot remains unknown; current registration + cannot attest a revision that did not accept a declaration. + """ + application = self.application + async with self._read(context) as connection: + record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) + return await AtomicMemoryEvidenceRepository().verify( + connection, + self.scope_id, + record.ref, + record.artifact.lineage.sources, + sources=application.artifacts.source_repository, + ) + async def get_many(self, artifact_ids: Sequence[str], *, context=None) -> tuple[AtomicMemoryRecord, ...]: """Read a caller's exact results in one snapshot after checking each resource.""" diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index a3a0dadd70..dd26ae25bb 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Explicit offline repair of only the current Atomic Memory projection.""" +"""Offline current-projection repair and verification of retained Atomic history.""" from __future__ import annotations import json -from collections.abc import Callable +from collections.abc import AsyncIterator, Callable from hashlib import sha256 +from itertools import groupby from time import perf_counter from pydantic import BaseModel, Field @@ -26,12 +27,17 @@ from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryRecord, AtomicMemoryStateValue from powercontext.builtin.artifacts.memory.canonical import canonical_json from powercontext.builtin.artifacts.search import analyze_text from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -42,7 +48,10 @@ drop_obsolete_atomic_memory_projection, load_atomic_memory_tags, ) -from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + ATOMIC_MEMORY_PROJECTION_FORMAT, +) from powercontext.builtin.persistence.atomic_memory_readiness import ( atomic_memory_readiness_issues, ensure_supported_atomic_memory_schema, @@ -50,6 +59,7 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.revision_metadata_schema import ensure_artifact_revision_metadata_schema +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE @@ -60,7 +70,7 @@ class AtomicMemoryProjectionRebuildReport(BaseModel): errors: tuple[str, ...] = () -async def _require_authority(connection: AsyncConnection) -> None: +async def _require_authority(connection: AsyncConnection, *, require_evidence: bool = True) -> None: await ensure_supported_atomic_memory_schema(connection) tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) required = { @@ -75,7 +85,7 @@ async def _require_authority(connection: AsyncConnection) -> None: raise AtomicMemoryIndexError( "authority-schema", "Atomic Memory authority is absent; complete initialization and history migration first" ) - issues = await atomic_memory_readiness_issues(connection, tables) + issues = await atomic_memory_readiness_issues(connection, tables, require_evidence=require_evidence) if issues: raise AtomicMemoryIndexError("migration-pending", "Atomic Memory migration is not ready: " + "; ".join(issues)) heads = ARTIFACT_HEADS_TABLE @@ -116,6 +126,111 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) +async def _evidence_history( + connection: AsyncConnection, + artifacts: ArtifactRepository, + batch_size: int, + sources: SourceRepository | None, + *, + selected_scope: str | None = None, +) -> AsyncIterator[AtomicMemoryEvidenceAnnotation]: + table = ARTIFACTS_TABLE + evidence = AtomicMemoryEvidenceRepository() + after: tuple[str, str, int] | None = None + while True: + statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( + table.c.family == AtomicMemory.family + ) + if selected_scope is not None: + statement = statement.where(table.c.scope_id == selected_scope) + if after is not None: + statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) + rows = ( + await connection.execute( + statement.order_by(table.c.scope_id, table.c.artifact_id, table.c.revision).limit(batch_size) + ) + ).all() + if not rows: + break + # Rows are ordered by scope, so one page holds contiguous per-scope groups. + for scope_id, group in groupby(rows, key=lambda row: str(row.scope_id)): + references = tuple( + ArtifactRef(family=AtomicMemory.family, artifact_id=str(row.artifact_id), revision=int(row.revision)) + for row in group + ) + loaded = await artifacts.get_many(connection, scope_id, references) + for reference, artifact in zip(references, loaded, strict=True): + yield await evidence.verify(connection, scope_id, reference, artifact.lineage.sources, sources=sources) + after = scope_id, reference.artifact_id, reference.revision + + +def _obsolete_evidence(): + projection = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + return ~exists( + select(ARTIFACTS_TABLE.c.artifact_id).where( + ARTIFACTS_TABLE.c.scope_id == projection.c.scope_id, + ARTIFACTS_TABLE.c.family == AtomicMemory.family, + ARTIFACTS_TABLE.c.artifact_id == projection.c.artifact_id, + ARTIFACTS_TABLE.c.revision == projection.c.revision, + ) + ) + + +async def rebuild_atomic_memory_evidence( + connection: AsyncConnection, + index: AtomicMemoryIndex, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, + guard: Callable[[], None] = lambda: None, + scope_id: str | None = None, +) -> None: + """Rebuild disposable revision facts without changing history or invoking models. + + The maintenance caller owns schema admission, writer exclusion and the transaction. + """ + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") + artifacts = ArtifactRepository((AtomicMemory,)) + async for annotation in _evidence_history(connection, artifacts, batch_size, sources, selected_scope=scope_id): + guard() + await index.project_evidence(connection, annotation) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + statement = table.delete().where(_obsolete_evidence()) + if scope_id is not None: + statement = statement.where(table.c.scope_id == scope_id) + await connection.execute(statement) + + +async def _verify_history( + connection: AsyncConnection, + index: AtomicMemoryIndex, + artifacts: ArtifactRepository, + batch_size: int, + sources: SourceRepository | None, + require_projection: bool, +) -> dict[str, int]: + revisions = neutral_sources = differing = 0 + async for annotation in _evidence_history(connection, artifacts, batch_size, sources): + revisions += 1 + neutral_sources += sum( + source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" + for source in annotation.sources + ) + if require_projection: + projected = await index.read_evidence_projection(connection, annotation.scope_id, annotation.artifact_ref) + differing += projected != annotation + counts = {"historical_revisions": revisions, "neutral_sources": neutral_sources} + if require_projection: + counts["evidence_projection_differences"] = differing + int( + await connection.scalar( + select(func.count()).select_from(ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE).where(_obsolete_evidence()) + ) + or 0 + ) + return counts + + async def _record( connection: AsyncConnection, artifacts: ArtifactRepository, @@ -183,17 +298,53 @@ async def _projection_differences( return differing +async def verify_atomic_memory_evidence( + connection: AsyncConnection, + index: AtomicMemoryIndex, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, +) -> AtomicMemoryProjectionRebuildReport: + """Validate exact revision facts independently of the deployed vector profile.""" + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") + await _require_authority(connection) + counts = await _verify_history(connection, index, ArtifactRepository((AtomicMemory,)), batch_size, sources, True) + differences = counts["evidence_projection_differences"] + return AtomicMemoryProjectionRebuildReport( + ready=not differences, + counts=counts, + errors=(f"{differences} evidence projections differ from retained history",) if differences else (), + ) + + async def verify_atomic_memory_current( - connection: AsyncConnection, index: AtomicMemoryIndex, /, *, batch_size: int = 100 + connection: AsyncConnection, + index: AtomicMemoryIndex, + /, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, + require_evidence: bool = False, + require_evidence_projection: bool = False, ) -> AtomicMemoryProjectionRebuildReport: - """Check current identities without taking business row locks. + """Check current projections and optionally each exact revision's retained evidence without business row locks. The caller owns a consistent read snapshot or an exclusive maintenance - transaction, including for identities created after migration. + transaction, including for identities created after migration. Frozen + pre-evidence migrations retain projection-only validation; current maintenance + and rebuild operations explicitly require evidence verification. """ - await _require_authority(connection) + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") + await _require_authority(connection, require_evidence=require_evidence or require_evidence_projection) artifacts = ArtifactRepository((AtomicMemory,)) + history_counts = ( + await _verify_history(connection, index, artifacts, batch_size, sources, require_evidence_projection) + if require_evidence or require_evidence_projection + else {} + ) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) columns = await connection.run_sync( @@ -206,6 +357,10 @@ async def verify_atomic_memory_current( for column in index.table.c ] errors: list[str] = [] + if history_counts.get("evidence_projection_differences"): + errors.append( + f"{history_counts['evidence_projection_differences']} evidence projections differ from retained history" + ) after = None active = 0 while batch := await _identity_batch(connection, after, batch_size): @@ -240,7 +395,7 @@ async def verify_atomic_memory_current( return AtomicMemoryProjectionRebuildReport( ready=not errors, profile_fingerprint=fingerprint, - counts={"active_rows": active, "obsolete_rows": obsolete}, + counts={"active_rows": active, "obsolete_rows": obsolete, **history_counts}, errors=tuple(errors), ) @@ -262,12 +417,16 @@ async def rebuild_atomic_memory_current( index: AtomicMemoryIndex, *, batch_size: int = 100, + sources: SourceRepository | None = None, guard: Callable[[], None] = lambda: None, ) -> AtomicMemoryProjectionRebuildReport: """Repair registered no-vector rows inside the maintenance owner's transaction. The caller validates the existing schema, owns backup/recovery and commits only after full Server validation. No initialization or external I/O occurs. + ``sources`` is the deployment's Source registry: passing it keeps this repair's + evidence verdict identical to the runtime read path. A maintenance bundle has no + host registry, so it omits this and admission uses ``BUILTIN_SOURCE_REGISTRY``. """ if index.capabilities.embedding_profile is not None: raise AtomicMemoryIndexError("embedding-profile", "Registered repair requires a no-vector profile") @@ -285,7 +444,15 @@ async def rebuild_atomic_memory_current( await publisher.remove(connection, scope_id, artifact_id) after = scope_id, artifact_id await connection.execute(index.table.delete().where(_obsolete_current(index))) - return await verify_atomic_memory_current(connection, index, batch_size=batch_size) + await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, sources=sources, guard=guard) + return await verify_atomic_memory_current( + connection, + index, + batch_size=batch_size, + sources=sources, + require_evidence=True, + require_evidence_projection=True, + ) async def rebuild_atomic_memory_projection( @@ -295,6 +462,7 @@ async def rebuild_atomic_memory_projection( maintenance_confirmed: bool, embedding_model: EmbeddingModel | None = None, batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: """Repair current without changing revisions, Family state or retained history. @@ -363,7 +531,15 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete - report = await verify_atomic_memory_current(connection, index, batch_size=batch_size) + await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, sources=sources) + report = await verify_atomic_memory_current( + connection, + index, + batch_size=batch_size, + sources=sources, + require_evidence=True, + require_evidence_projection=True, + ) return report.model_copy( update={ "counts": { @@ -380,6 +556,8 @@ async def rebuild_atomic_memory_projection( __all__ = [ "AtomicMemoryProjectionRebuildReport", "rebuild_atomic_memory_current", + "rebuild_atomic_memory_evidence", "rebuild_atomic_memory_projection", "verify_atomic_memory_current", + "verify_atomic_memory_evidence", ] diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index d37c3bf674..db9aaa72e9 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -226,7 +226,7 @@ ) from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError -from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey +from powercontext.sources import AdapterSourceDefinition, Source, SourceDefinitionRegistry, SourceProjectionKey if TYPE_CHECKING: from pydantic_ai.models import Model @@ -962,7 +962,20 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( - definition is not expected for definition, expected in zip(definitions, builtin_definitions, strict=True) + # A host may replace only the evidence declaration while keeping the + # builtin adapter, version and projections. `memory_evidence` is + # deliberately excluded: it is the dimension this registration adds. + not ( + definition is expected + or ( + type(definition) is AdapterSourceDefinition + and type(expected) is AdapterSourceDefinition + and definition.adapter is expected.adapter + and definition.version == expected.version + and definition.projections is expected.projections + ) + ) + for definition, expected in zip(definitions, builtin_definitions, strict=True) ) ): raise BuiltinConfigurationError("artifact-processing-source-registry") diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 03aebe12aa..52fb49ddbd 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -1422,6 +1422,8 @@ async def handoff_scope_ids(self) -> tuple[str, ...]: async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> None: """Rebuild target-local search projections after a logical restore.""" + from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_evidence + for scope_id in scope_ids: async with self.database.transaction() as connection: await self.atomic_memory_index.initialize(connection) @@ -1467,6 +1469,9 @@ async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> N ) if projected != rebuilt: raise RuntimeError("restored Atomic Memory projection is incomplete") # noqa: TRY003 + await rebuild_atomic_memory_evidence( + connection, self.atomic_memory_index, sources=self.repositories.sources, scope_id=scope_id + ) topic_heads = tuple( ( await connection.execute( diff --git a/src/powercontext/errors.py b/src/powercontext/errors.py index 1ad41604c4..c45dd54889 100644 --- a/src/powercontext/errors.py +++ b/src/powercontext/errors.py @@ -25,6 +25,17 @@ class SourceError(PowerContextError): """Base exception for Source adapter and access failures.""" +class AtomicMemoryEvidenceError(PowerContextError): + """Raised when retained evidence no longer matches the exact accepted material. + + A caller can distinguish this integrity failure from retrieval or projection + errors through the stable :class:`PowerContextError` family. + """ + + def __init__(self) -> None: + super().__init__("Historical evidence does not match the exact Source material") + + class SourceNotFoundError(SourceError, LookupError): """Raised when a Source object is absent from a catalog.""" diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index d77b027214..b0d24635c7 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -773,6 +773,9 @@ async def _atomic_memory_maintenance( if action in {"apply", "rebuild-projection"}: embedding_model, _readiness = await _embedding_models(inference, resources, None) if action == "rebuild-projection": + # This command reads deployment configuration, not host code, so + # evidence admission uses BUILTIN_SOURCE_REGISTRY. Hosts with native + # Definitions pass the runtime registry to the Python API instead. report = await rebuild_atomic_memory_projection( profile.database, index, diff --git a/src/powercontext/server/logging.py b/src/powercontext/server/logging.py index c8f91e808a..d30933bba5 100644 --- a/src/powercontext/server/logging.py +++ b/src/powercontext/server/logging.py @@ -105,6 +105,7 @@ "verdict", "final_action", "fallback_reason", + "reason", "candidate_count", "evidence_count", "trace_id", @@ -208,7 +209,7 @@ def filter(self, record: logging.LogRecord) -> bool: if field not in self._identity and field not in {"request_id", "trace_id", "span_id"} ) if isinstance(event, str) and event.startswith("code."): - fields += _CODE_FIELDS + fields = tuple(dict.fromkeys((*fields, *_CODE_FIELDS))) record.processing_context = "".join( f" {field}={json.dumps(value, ensure_ascii=False)}" for field in fields diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 91e1a27b08..5469244670 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,6 +30,7 @@ SourceDefinitionRegistry, SourceProjection, ) +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef from powercontext.sources.observations import ( SourceDefinitionManifest, @@ -53,6 +54,7 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", + "MemoryEvidenceDeclaration", "Source", "SourceAdapter", "SourceCatalog", @@ -60,6 +62,7 @@ "SourceDefinition", "SourceDefinitionManifest", "SourceDefinitionRegistry", + "SourceEvidence", "SourceMaterialization", "SourceObservation", "SourceProjection", diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d9c54572dc..0f3b9612f1 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -16,8 +16,10 @@ from __future__ import annotations +import logging from collections.abc import Iterable, Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -34,6 +36,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter +from powercontext.sources.evidence import MemoryEvidenceDeclaration, log_evidence_neutralized from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -42,6 +45,7 @@ _JSON_VALUE = TypeAdapter(JsonValue) _AnySourceAdapter = SourceAdapter[Any, Any, Any] +_NEUTRAL_MEMORY_EVIDENCE = MemoryEvidenceDeclaration() class SourceProjection(Protocol[SourceT]): @@ -60,6 +64,7 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] + memory_evidence: MemoryEvidenceDeclaration | None @dataclass(frozen=True, slots=True) @@ -69,6 +74,7 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () + memory_evidence: MemoryEvidenceDeclaration | None = None @property def input_class(self) -> type[InputT]: @@ -93,14 +99,31 @@ async def read(self, source: SourceT, /) -> ValueT_co: class SourceDefinitionRegistry: - """Provide one immutable routing view for Source persistence and consumers.""" - - def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: + """Provide one immutable routing and trusted declaration view. + + Native Definitions optionally supply ``memory_evidence``. Deployment-owned + ``remote_declarations`` keys are exact ``(name, version, fingerprint)`` + manifest identities, never worker payload or projection values. Declarations + are validated and copied at construction; invalid or unavailable values are + neutral and never prevent registration of an otherwise valid Definition. + Only host code constructing this registry supplies ``remote_declarations``; + no configuration file entry exists, because the keys must match manifests + that the deployment itself registered. + """ + + def __init__( + self, + definitions: Iterable[_AnySourceDefinition], + /, + *, + remote_declarations: Mapping[tuple[str, str, str], MemoryEvidenceDeclaration] | None = None, + ) -> None: by_input: dict[type[object], _AnySourceDefinition] = {} by_source: dict[type[Source], _AnySourceDefinition] = {} by_name: dict[str, _AnySourceDefinition] = {} projections: dict[type[Source], Mapping[SourceProjectionKey, SourceProjection[Any]]] = {} registered: list[_AnySourceDefinition] = [] + memory_evidence: dict[type[Source], MemoryEvidenceDeclaration] = {} for definition in definitions: input_class, source_class = _validate_definition(definition) @@ -122,6 +145,11 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: by_source[source_class] = definition by_name[definition.name] = definition projections[source_class] = projection_routes + try: + declaration = getattr(definition, "memory_evidence", None) + except Exception: # Declaration providers may be unavailable; routing remains valid. + declaration = None + memory_evidence[source_class] = _snapshot_memory_evidence(declaration) registered.append(definition) self._definitions = tuple(registered) @@ -129,6 +157,10 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: self._by_source = by_source self._by_name = by_name self._projections = projections + self._memory_evidence = MappingProxyType(memory_evidence) + self._remote_declarations = MappingProxyType({ + key: _snapshot_memory_evidence(value) for key, value in (remote_declarations or {}).items() + }) @classmethod def from_adapters(cls, adapters: Iterable[_AnySourceAdapter], /) -> SourceDefinitionRegistry: @@ -161,6 +193,23 @@ def definition_for_source(self, source: object, /) -> _AnySourceDefinition: ) return definition + def memory_evidence_for_source(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Read the frozen trusted declaration using the exact registered route.""" + + from powercontext.sources.observations import SourceObservation + + if isinstance(source, SourceObservation): + return self.memory_evidence_for_manifest( + source.source_type, source.definition_version, source.definition_fingerprint + ) + self.definition_for_source(source) + return self._memory_evidence[type(source)] + + def memory_evidence_for_manifest(self, name: str, version: str, fingerprint: str, /) -> MemoryEvidenceDeclaration: + """Return only deployment-configured evidence for an exact manifest.""" + + return self._remote_declarations.get((name, version, fingerprint), _NEUTRAL_MEMORY_EVIDENCE) + async def resolve(self, value: object, /) -> Source: input_class = type(value) try: @@ -203,6 +252,19 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error +def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: + if value is None: + return _NEUTRAL_MEMORY_EVIDENCE + try: + # Revalidate even constructed models, and never retain caller-owned data. + if isinstance(value, MemoryEvidenceDeclaration): + value = value.model_dump(mode="python") + return MemoryEvidenceDeclaration.model_validate(value) + except (TypeError, ValueError): + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) + return _NEUTRAL_MEMORY_EVIDENCE + + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py new file mode 100644 index 0000000000..d2bcb99059 --- /dev/null +++ b/src/powercontext/sources/evidence.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed declarations and exact material facts accepted at Source registration.""" + +from __future__ import annotations + +import logging +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext._logging import log_safely +from powercontext.sources.models import SourceMaterialization, SourceRef + +_EVIDENCE_LOGGER = logging.getLogger(__name__) + +EvidenceNeutralReason = Literal[ + "schema_mismatch", + "digest_mismatch", + "no_manifest", + "decode_failed", + "legacy_unverifiable", + "invalid_declaration", + "manifest_conflict", + "definition_missing", + "evidence_mismatch", +] +"""Bounded reasons why accepted evidence resolved to neutral. + +The vocabulary is operational only: it names which retained fact could not be +attested, never the material, declaration body, or requested content. +""" + +EvidenceNeutralUnit = Literal["source", "declaration", "manifest", "revision"] + + +def log_evidence_neutralized( + reason: EvidenceNeutralReason, + *, + unit: EvidenceNeutralUnit, + level: int = logging.INFO, +) -> None: + """Emit one content-free record for a neutralized evidence fact.""" + + log_safely( + _EVIDENCE_LOGGER, + level, + "Memory evidence resolved to neutral", + extra={ + "event": "atomic_memory.evidence.neutral", + "operation": "atomic_memory.evidence", + "unit": unit, + "outcome": "neutral", + "reason": reason, + }, + ) + + +class MemoryEvidenceDeclaration(BaseModel): + """A trusted adapter's claim, not a score, permission or verification engine. + + ``declaration_version`` identifies this contract, independently of the + Definition version and Artifact content revision. Missing or unsupported + declarations resolve to ``untrusted`` / ``unknown``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + authority: Literal["untrusted", "user_asserted", "repository_attested", "system_attested"] = "untrusted" + verification: Literal["unknown", "verified", "not_verified"] = "unknown" + declaration_version: Literal["1"] = "1" + + +class SourceEvidence(BaseModel): + """Retain one exact Source material binding without duplicating its body. + + The digest checks equality of canonical Source JSON, excluding only the + server-owned ContentSource handoff receipt witness. It authenticates neither + the Source nor its declaration. Each Source retains its own declaration. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_ref: SourceRef + materialization: SourceMaterialization + material_digest: str + definition_name: str + definition_version: str + manifest_fingerprint: str | None = None + declaration: MemoryEvidenceDeclaration = Field(default_factory=MemoryEvidenceDeclaration) + + +__all__ = ["MemoryEvidenceDeclaration", "SourceEvidence"] diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6e2cd48df6..0f1a4a26fe 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -60,6 +60,8 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" + model_config = ConfigDict(populate_by_name=True) + name: str definition_version: str = "1" materialization: SourceMaterialization diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 576e29fb79..e7a070eb13 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,10 +17,21 @@ from __future__ import annotations import hashlib -from typing import Any, Literal +from types import UnionType +from typing import Any, Literal, Union, get_args, get_origin import rfc8785 -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator +from pydantic import ( + AliasChoices, + AliasPath, + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + field_validator, + model_validator, +) from powercontext.errors import ( InvalidSourceDefinitionError, @@ -161,9 +172,28 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build the immutable declaration transported by a remote worker.""" + """Build a declaration matching the canonical remote payload representation.""" + + return _manifest_for_definition(definition, transport=True) + - source_schema = _json_object(definition.source_class.model_json_schema()) +def _manifest_for_definition( + definition: SourceDefinition[Any, Any, Any], + /, + *, + transport: bool, +) -> SourceDefinitionManifest: + if transport and _validation_schema_matches_payload(definition.source_class): + source_schema = _json_object(definition.source_class.model_json_schema()) + elif transport: + source_schema = _json_object( + definition.source_class.model_json_schema( + mode="serialization", + by_alias=bool(definition.source_class.model_config.get("serialize_by_alias")), + ) + ) + else: + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -215,6 +245,101 @@ def project_source_for_transport( ) +def _validation_schema_matches_payload( + source_class: type[BaseModel], + seen: set[type[BaseModel]] | None = None, +) -> bool: + """Return whether validation-mode keys accept the serialized payload.""" + + visited = set() if seen is None else seen + if source_class in visited: + return True + visited.add(source_class) + serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) + # Preserve existing permissive manifests; optional renamed keys only fail when extras are forbidden. + forbid_extra = source_class.model_config.get("extra") == "forbid" + for name, field in source_class.model_fields.items(): + if field.exclude: + continue + if field.is_required() or forbid_extra: + serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name + if _validation_schema_key(name, field) != serialization_key: + return False + # A null default means this field can reach the payload as null, so a renamed + # shape below it need not appear and cannot justify replacing a manifest a + # deployment already accepted. Populating one instead requires the nested + # model to serialize by alias, which keeps the validation keys authoritative, + # or a new Definition version. + if field.default is None and not forbid_extra: + continue + if not _validation_schema_matches_annotation(field.annotation, visited): + return False + return True + + +def _validation_schema_matches_annotation(annotation: Any, seen: set[type[BaseModel]]) -> bool: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return _validation_schema_matches_payload(annotation, seen) + arguments = get_args(annotation) + if get_origin(annotation) in (Union, UnionType): + return _union_validation_covers_payloads(arguments, seen) + return all(_validation_schema_matches_annotation(argument, seen) for argument in arguments) + + +def _union_validation_covers_payloads(arguments: tuple[Any, ...], seen: set[type[BaseModel]]) -> bool: + # Every possible output needs coverage, not merely one compatible branch. + for branch in arguments: + if _validation_schema_matches_annotation(branch, set(seen)): + continue + if not isinstance(branch, type) or not issubclass(branch, BaseModel): + return False + serialized = branch.model_json_schema( + mode="serialization", by_alias=bool(branch.model_config.get("serialize_by_alias")) + ) + if not any( + isinstance(candidate, type) + and issubclass(candidate, BaseModel) + and _same_flat_schema(serialized, candidate.model_json_schema()) + for candidate in arguments + ): + return False + return True + + +def _same_flat_schema(left: dict[str, Any], right: dict[str, Any]) -> bool: + """Prove flat model equivalence without interpreting references or nested schemas.""" + supported = {"title", "type", "properties", "required", "additionalProperties"} + if left.keys() - supported or right.keys() - supported: + return False + left_properties = left.get("properties", {}) + right_properties = right.get("properties", {}) + if left_properties.keys() != right_properties.keys(): + return False + for name, schema in left_properties.items(): + other = right_properties[name] + # Only primitive properties: no refs, unions, containers or nested schemas. + if schema.get("type") not in {"string", "integer", "number", "boolean", "null"}: + return False + if {key: value for key, value in schema.items() if key != "title"} != { + key: value for key, value in other.items() if key != "title" + }: + return False + return all(left.get(key) == right.get(key) for key in supported - {"title", "properties"}) + + +def _validation_schema_key(name: str, field: Any) -> str | None: + alias = field.validation_alias + if isinstance(alias, str): + return alias + if isinstance(alias, AliasChoices): + return next((choice for choice in alias.choices if isinstance(choice, str)), None) + if isinstance(alias, AliasPath): + return name # Pydantic keeps the field name for path-based validation. + if alias is not None: + return None + return field.alias if field.alias is not None else name + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_atomic_description_vector_layout.py b/tests/builtin/persistence/test_atomic_description_vector_layout.py index 1c41ce29e4..534a838205 100644 --- a/tests/builtin/persistence/test_atomic_description_vector_layout.py +++ b/tests/builtin/persistence/test_atomic_description_vector_layout.py @@ -18,6 +18,7 @@ import shutil import sqlite3 from contextlib import closing +from pathlib import Path import pytest @@ -42,7 +43,7 @@ def test_managed_sqlite_vector_schema_includes_registered_nullable_description(t database = tmp_path / "vector.db" runner = SQLiteMigrationRunner(database, bundle, configuration=configuration) result = runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) - assert result.revision == "v1_5_0_001" + assert result.revision == "v1_5_0_003" with closing(sqlite3.connect(database)) as connection: columns = {row[1]: row for row in connection.execute("PRAGMA table_info(pc_atomic_memory_current)")} assert columns["description"][2:4] == ("VARCHAR(200)", 0) @@ -54,19 +55,18 @@ def test_mysql_vector_admission_retains_description_and_exact_previous_revision( plain = server_bundle() vector = server_bundle(embedding_profile=profile) assert vector.vector_profile is not None + historical_revisions = plain.revisions[: plain.revisions.index("v1_4_1_005") + 1] inventory = {"tables": {}, "views": [], "triggers": []} for resource in json.loads((plain.directory / "manifest.json").read_text())["mysql_state_layouts"]: - if resource.endswith("mysql-v1_5_0_001.json"): - continue - for entry in json.loads((plain.directory / resource).read_text())["revisions"].values(): + for revision, entry in json.loads((plain.directory / resource).read_text())["revisions"].items(): + if revision not in historical_revisions: + continue for change in entry["changes"]: for name, table in change.items(): if table is None: del inventory["tables"][name] else: inventory["tables"][name] = table - # Inspect the description revision before the subsequent Source registration table. - del inventory["tables"]["pc_source_write_plugins"] for name, sql in vector.vector_profile.mysql_tables().items(): if name == "pc_atomic_memory_current": sql = sql.replace( @@ -103,21 +103,42 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm shutil.copytree(current.snapshot_directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - removed_revisions = ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") - for revision in removed_revisions: - for resource in manifest["revision_resources"].pop(revision): - (directory / resource).unlink() - for script in (directory / "versions").glob(f"{revision}_*.py"): - script.unlink() - (directory / "mysql/versions" / f"{revision}.py").unlink() + appended = current.revisions[current.revisions.index("v1_4_1_004") + 1 :] + removed_resources = {resource for revision in appended for resource in manifest["revision_resources"].pop(revision)} + removed_steps = [step for step in manifest["domain_steps"] if step["revision"] in appended] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in appended] + for step in removed_steps: + removed_resources.update((step["module"], *step.get("resources", ()))) + retained_resources = {resource for resources in manifest["revision_resources"].values() for resource in resources} + for step in manifest["domain_steps"]: + retained_resources.update((step["module"], *step.get("resources", ()))) + # The bundle substitutes its vector domain module and profile at load time. + retained_resources.update({"server_vector_domain.py", "profiles/vector.json"}) + for name in appended: + revision = current.script.get_revision(name) + removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed_resources.add(f"mysql/versions/{name}.py") + for resource in removed_resources - retained_resources: + (directory / resource).unlink() + for revision in appended: del manifest["sqlite_fingerprints"][revision] del manifest["domain_readiness"][revision] - manifest["mysql_state_layouts"].remove(f"schemas/mysql-{revision}.json") - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in removed_revisions] - (directory / "artifact_storage_readiness.py").unlink() - manifest["affected_objects"].remove("pc_source_write_plugins") + manifest.get("mysql_wrappers", {}).pop(revision, None) + manifest["mysql_state_layouts"] = [ + resource for resource in manifest["mysql_state_layouts"] if resource not in removed_resources + ] + appended_tables = { + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", + "pc_source_evidence_bindings", + "pc_source_write_plugins", + } + manifest["affected_objects"] = [name for name in manifest["affected_objects"] if name not in appended_tables] manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") previous = MigrationBundle(directory, embedding_profile=profile) + assert previous.head == "v1_4_1_004" + assert previous.checksums == {revision: current.checksums[revision] for revision in previous.revisions} configuration = { "embedding_enabled": profile is not None, "processing_manifest": canonical_processing_manifest(BuiltinConfig()), @@ -130,11 +151,16 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm owner_record = ("original-operation", "original-deployment", "original-scope", 0, '{"principal":"原始身份"}') with closing(sqlite3.connect(database)) as connection, connection: connection.execute("INSERT INTO pc_access_owner_recovery VALUES (?,?,?,?,?)", owner_record) - original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) + original_evidence_bytes = old.evidence_path.read_bytes() + original_evidence = json.loads(original_evidence_bytes) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration) plan = upgraded.plan() - assert plan.source_revision == "v1_4_1_004" and plan.revisions == removed_revisions - assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_001" + assert plan.source_revision == "v1_4_1_004" + assert plan.target_revision == "v1_5_0_003" + assert plan.revisions == ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") + assert plan.domain_steps == ("artifact_storage", "atomic_memory_evidence", "atomic_memory_evidence_projection") + assert upgraded.evidence_path.read_bytes() == original_evidence_bytes + assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_003" assert upgraded.verify().state == "ready" evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) for revision, checksum in original_evidence["release_checksums"].items(): diff --git a/tests/builtin/persistence/test_atomic_evidence_schema_copies.py b/tests/builtin/persistence/test_atomic_evidence_schema_copies.py new file mode 100644 index 0000000000..49247b2394 --- /dev/null +++ b/tests/builtin/persistence/test_atomic_evidence_schema_copies.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keep the runtime evidence revision and the Server bundle on identical DDL.""" + +from __future__ import annotations + +import json +from pathlib import Path + +_PERSISTENCE = Path(__file__).resolve().parents[3] / "src/powercontext/builtin/persistence" +_RUNTIME = _PERSISTENCE / "migrations/resources/schemas/atomic_memory_evidence_v1.json" +_SERVER = _PERSISTENCE / "migrations/server_resources/schemas" + + +def test_runtime_and_server_evidence_ddl_are_identical() -> None: + # Runtime readiness compares SQLite `sqlite_master` text exactly, so a drift + # between these two producer copies fails managed readiness outside CI. + runtime = json.loads(_RUNTIME.read_text(encoding="utf-8")) + sqlite = json.loads((_SERVER / "sqlite-v1_5_0_002.json").read_text(encoding="utf-8")) + mysql = json.loads((_SERVER / "mysql-v1_5_0_002.json").read_text(encoding="utf-8")) + + assert runtime["sqlite"] == sqlite["create"] + assert runtime["mysql"] == mysql["revisions"]["v1_5_0_002"]["statements"] diff --git a/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py new file mode 100644 index 0000000000..6e677d1d05 --- /dev/null +++ b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py @@ -0,0 +1,268 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The evidence revision must accept maintenance history written by upstream.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle +from powercontext.builtin.persistence.migrations.deployment import server_bundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest + +EVIDENCE_REVISION = "v1_5_0_002" +PROJECTION_REVISION = "v1_5_0_003" +EVIDENCE_TABLES = { + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", + "pc_source_evidence_bindings", +} + +# Recorded from the unmodified aedf7beb upstream bundle, not recomputed from +# resources under test. Vector checksums bind the exact profile used below. +UPSTREAM_RELEASE_CHECKSUMS = { + False: { + "s1300": "ed030becdb3d2aeffd289440fd492fa31c7e53b6c3f8e30aa7119835353f1d44", + "s1400": "ab457bcd03e7bf7e7003342104947b031dcfd30e121e836df3af6eddc7dce690", + "s1401": "bcd58485ec5d1945b18a03335346aaef2eebe403fbfc62ca73e8d56059035e85", + "s1402": "c0642c4535e2056fd9c2997a70621c26f6c46aa3d9d6cc4dedd1656bc0397c24", + "v1_4_1_004": "3d3e3a63024d0c4f7efae425c13137db32772f5afb5686148cbd0a5a6d434c54", + }, + True: { + "s1300": "2a20255dca66ad183260dcf40dffdf493eae0e160000e6aa3d47bbb20ee92310", + "s1400": "08013a6084b7cfedda54ecc10347621b4a5460fbbf1935cdd25791799d185158", + "s1401": "3bc6adcd391ccc04fac21acd2ebcdf372295513024452e6ad613e1620c72d06b", + "s1402": "e262e4ef25c43b65ec356e3bd27ccfa46c0100e1689aabf31756c205c9098b23", + "v1_4_1_004": "8340a6331e2fc81a44f79cc7848feb209ac61338ff565e2d42c5a607e9e61c5b", + }, +} + + +def previous_upstream_bundle(directory, embedding): + """Drop registrations appended after the frozen upstream owner-recovery revision. + + Historical checksums below are independent upstream constants: copying the + installed resources is not allowed to silently redefine the old release. + No git checkout, network access or runtime table initializer is required. + """ + + current = server_bundle(embedding_profile=embedding) + shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + appended = current.revisions[current.revisions.index("v1_4_1_004") + 1 :] + removed_resources = {resource for revision in appended for resource in manifest["revision_resources"].pop(revision)} + removed_steps = [step for step in manifest["domain_steps"] if step["revision"] in appended] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in appended] + for step in removed_steps: + removed_resources.update((step["module"], *step.get("resources", ()))) + retained_resources = {resource for resources in manifest["revision_resources"].values() for resource in resources} + for step in manifest["domain_steps"]: + retained_resources.update((step["module"], *step.get("resources", ()))) + # The bundle substitutes its vector domain module and profile at load time. + retained_resources.update({"server_vector_domain.py", "profiles/vector.json"}) + for name in appended: + revision = current.script.get_revision(name) + removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed_resources.add(f"mysql/versions/{name}.py") + for resource in removed_resources - retained_resources: + (directory / resource).unlink() + for revision in appended: + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest["mysql_state_layouts"] = [ + resource for resource in manifest["mysql_state_layouts"] if resource not in removed_resources + ] + for revision in appended: + manifest.get("mysql_wrappers", {}).pop(revision, None) + manifest["affected_objects"] = [ + name + for name in manifest["affected_objects"] + if name not in EVIDENCE_TABLES and name != "pc_source_write_plugins" + ] + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return MigrationBundle(directory, embedding_profile=embedding) + + +def configuration(embedding): + result = { + "processing_manifest": canonical_processing_manifest(BuiltinConfig()), + "embedding_enabled": embedding is not None, + } + if embedding is not None: + result["embedding_profile"] = embedding.model_dump(mode="json") + return result + + +def apply(runner): + return runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) + + +@pytest.mark.parametrize("vectors", [False, True], ids=["lexical", "vectors"]) +def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, vectors): + embedding = EmbeddingProfile(profile_id="evidence-upgrade", model="fixture", dimension=3) if vectors else None + config = configuration(embedding) + expected = UPSTREAM_RELEASE_CHECKSUMS[vectors] + previous = previous_upstream_bundle(tmp_path / "upstream-bundle", embedding) + assert previous.head == "v1_4_1_004" + assert previous.checksums == expected + + database = tmp_path / "managed.db" + old = SQLiteMigrationRunner(database, previous, configuration=config) + assert apply(old).revision == "v1_4_1_004" + historical_time = "2026-10-23 12:34:56.123456" + historical_content = b'{"history":"unchanged"}' + with closing(sqlite3.connect(database)) as connection, connection: + connection.execute( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('original','title','summary','original','title','summary',1)" + ) + connection.execute( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content,created_at) " + "VALUES ('original','custom','preserved',1,?,?)", + (historical_content, historical_time), + ) + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_schema WHERE type='table'")} + assert not EVIDENCE_TABLES & tables + assert old.verify().state == "ready" + old_evidence_bytes = old.evidence_path.read_bytes() + old_evidence = json.loads(old_evidence_bytes) + assert old_evidence["state"] == "complete" + assert old_evidence["release_checksums"] == expected + + current = server_bundle(embedding_profile=embedding) + assert {revision: current.checksums[revision] for revision in expected} == expected + upgraded = SQLiteMigrationRunner(database, current, configuration=config) + plan = upgraded.plan() + assert plan.state == "migration_required" + assert plan.source_revision == "v1_4_1_004" + assert plan.target_revision == PROJECTION_REVISION + assert plan.revisions == ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", EVIDENCE_REVISION, PROJECTION_REVISION) + assert plan.domain_steps == ( + "artifact_storage", + "atomic_memory_evidence", + "atomic_memory_evidence_projection", + ) + assert upgraded.evidence_path.read_bytes() == old_evidence_bytes + result = upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert result.changed + assert result.revision == PROJECTION_REVISION + assert result.backup_state == "completed" + + # Reconstruct both bundle and runner: verification must use retained on-disk + # maintenance history, not an in-memory object from the upgrade operation. + reopened = SQLiteMigrationRunner(database, server_bundle(embedding_profile=embedding), configuration=config) + completed_evidence_bytes = reopened.evidence_path.read_bytes() + completed_evidence = json.loads(completed_evidence_bytes) + assert completed_evidence["state"] == "complete" + assert {revision: completed_evidence["release_checksums"][revision] for revision in expected} == expected + assert set(completed_evidence["release_checksums"]) == { + *expected, + "v1_4_1_005", + "v1_4_1_006", + "v1_5_0_001", + EVIDENCE_REVISION, + PROJECTION_REVISION, + } + ready = reopened.plan() + assert ready.state == "ready" + assert ready.revisions == () + assert reopened.verify().revision == PROJECTION_REVISION + assert not apply(reopened).changed + assert reopened.evidence_path.read_bytes() == completed_evidence_bytes + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT scope_id,title,summary,version FROM pc_scopes").fetchall() == [ + ("original", "title", "summary", 1) + ] + assert connection.execute( + "SELECT scope_id,family,artifact_id,revision,content,created_at FROM pc_artifacts" + ).fetchall() == [("original", "custom", "preserved", 1, historical_content, historical_time)] + assert connection.execute("SELECT schema_version FROM pc_atomic_memory_evidence_schema").fetchall() == [ + ("p0005",) + ] + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence_history").fetchone() == (0,) + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence").fetchone() == (0,) + assert connection.execute("SELECT COUNT(*) FROM pc_source_evidence_bindings").fetchone() == (0,) + if embedding is not None: + stored_profile = connection.execute("SELECT embedding_profile FROM pc_server_storage_profile").fetchone() + assert json.loads(stored_profile[0]) == embedding.model_dump(mode="json") + + # Removing only the marker preserves the schema fingerprint. Live validation + # must still reject incomplete evidence readiness and must not repair it. + with closing(sqlite3.connect(database)) as connection, connection: + connection.execute("DELETE FROM pc_atomic_memory_evidence_schema") + with pytest.raises(MigrationError) as caught: + reopened.verify() + assert caught.value.code == "domain_validation_failed" + assert reopened.evidence_path.read_bytes() == completed_evidence_bytes + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT schema_version FROM pc_atomic_memory_evidence_schema").fetchall() == [] + + +@pytest.mark.parametrize("vectors", [False, True], ids=["lexical", "vectors"]) +def test_evidence_projection_upgrade_preserves_registered_evidence_revision(tmp_path, vectors): + embedding = EmbeddingProfile(profile_id="evidence-upgrade", model="fixture", dimension=3) if vectors else None + current = server_bundle(embedding_profile=embedding) + directory = tmp_path / "evidence-bundle" + shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + removed = manifest["revision_resources"].pop(PROJECTION_REVISION) + removed.append("atomic_memory_evidence_projection.py") + revision = current.script.get_revision(PROJECTION_REVISION) + removed.append(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed.append(f"mysql/versions/{PROJECTION_REVISION}.py") + for resource in removed: + (directory / resource).unlink() + del manifest["sqlite_fingerprints"][PROJECTION_REVISION] + del manifest["domain_readiness"][PROJECTION_REVISION] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != PROJECTION_REVISION] + manifest["mysql_state_layouts"] = [name for name in manifest["mysql_state_layouts"] if name not in removed] + manifest["affected_objects"].remove("pc_atomic_memory_evidence") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + previous = MigrationBundle(directory, embedding_profile=embedding) + assert previous.head == EVIDENCE_REVISION + assert {revision: current.checksums[revision] for revision in previous.revisions} == previous.checksums + + database = tmp_path / "adjacent.db" + config = configuration(embedding) + old = SQLiteMigrationRunner(database, previous, configuration=config) + assert apply(old).revision == EVIDENCE_REVISION + before = old.evidence_path.read_bytes() + upgraded = SQLiteMigrationRunner(database, current, configuration=config) + plan = upgraded.plan() + assert plan.revisions == (PROJECTION_REVISION,) + assert plan.domain_steps == ("atomic_memory_evidence_projection",) + assert old.evidence_path.read_bytes() == before + assert ( + upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == PROJECTION_REVISION + ) + assert upgraded.verify().state == "ready" + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence").fetchone() == (0,) + assert connection.execute("SELECT schema_version FROM pc_atomic_memory_evidence_schema").fetchall() == [ + ("p0005",) + ] diff --git a/tests/builtin/persistence/test_evidence_neutralization_logging.py b/tests/builtin/persistence/test_evidence_neutralization_logging.py new file mode 100644 index 0000000000..d77175ccbd --- /dev/null +++ b/tests/builtin/persistence/test_evidence_neutralization_logging.py @@ -0,0 +1,166 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Neutral evidence facts report a bounded, content-free operational reason.""" + +from __future__ import annotations + +import asyncio +import json +import logging + +import pytest +from pydantic import field_serializer, field_validator +from sqlalchemy import update + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.server.logging import JsonFormatter, OperationalContextFilter +from powercontext.sources import ( + AdapterSourceDefinition, + MemoryEvidenceDeclaration, + SourceDefinitionRegistry, + SourceMaterialization, + project_source_for_transport, +) +from tests.builtin.persistence.contract import NoteAdapter, NoteSource, repository_profile + + +def _neutral_records(caplog) -> list[logging.LogRecord]: + return [record for record in caplog.records if getattr(record, "event", None) == "atomic_memory.evidence.neutral"] + + +def test_undecodable_snapshot_reports_decode_failed(caplog) -> None: + reference = ArtifactRef(family="atomic-memory", artifact_id="memory", revision=1) + + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + assert AtomicMemoryEvidenceRepository._decode("scope", reference, b"\xffnot-json") is None + + record = _neutral_records(caplog)[-1] + assert record.getMessage() == "Memory evidence resolved to neutral" + assert ( + getattr(record, "operation", None), + getattr(record, "unit", None), + getattr(record, "outcome", None), + getattr(record, "reason", None), + ) == ( + "atomic_memory.evidence", + "declaration", + "neutral", + "decode_failed", + ) + + +def test_neutral_reason_is_serialized_without_evidence_content(caplog) -> None: + reference = ArtifactRef(family="atomic-memory", artifact_id="memory", revision=1) + + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + AtomicMemoryEvidenceRepository._decode("scope", reference, b"\xffnot-json") + + record = _neutral_records(caplog)[-1] + OperationalContextFilter().filter(record) + payload = json.loads(JsonFormatter().format(record)) + + assert payload["event"] == "atomic_memory.evidence.neutral" + assert (payload["operation"], payload["unit"], payload["outcome"], payload["reason"]) == ( + "atomic_memory.evidence", + "declaration", + "neutral", + "decode_failed", + ) + assert set(payload) <= { + "timestamp", + "level", + "logger", + "message", + "event", + "operation", + "unit", + "outcome", + "reason", + "request_id", + "trace_id", + "span_id", + "service_name", + "service_version", + "service_role", + "service_instance_id", + } + + +def test_observation_fingerprint_mismatch_reports_digest_mismatch(caplog) -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry.from_adapters((NoteAdapter(),)) + source = NoteSource(name="observation", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + definition = registry.definition_for_name("note") + async with repository_profile() as (profile, _), profile.database.transaction() as connection: + await SourceDefinitionManifestRepository().register_native(connection, definition) + observation = project_source_for_transport(registry, source).model_copy( + update={"definition_fingerprint": "sha256:" + "0" * 64} + ) + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + await SourceRepository(registry).add(connection, "scope", observation) + records = _neutral_records(caplog) + assert [getattr(record, "reason", None) for record in records] == ["digest_mismatch"] + assert records[0].levelno == logging.WARNING + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("legacy_binding", [False, True]) +def test_schema_neutral_evidence_does_not_report_digest_corruption(caplog, legacy_binding: bool) -> None: + class SerializedNote(NoteSource): + @field_serializer("body") + def serialize_body(self, value: str): + return {"text": value} + + @field_validator("body", mode="before") + @classmethod + def deserialize_body(cls, value): + return value["text"] if isinstance(value, dict) else value + + class SerializedNoteAdapter(NoteAdapter): + source_class = SerializedNote + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + definition = AdapterSourceDefinition(SerializedNoteAdapter(), memory_evidence=declaration) + repository = SourceRepository(SourceDefinitionRegistry((definition,))) + source = SerializedNote( + name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained text." + ) + async with repository_profile() as (profile, _), profile.database.transaction() as connection: + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + stored = await repository.add(connection, "scope", source) + assert [getattr(record, "reason", None) for record in _neutral_records(caplog)] == ["schema_mismatch"] + if legacy_binding: + manifest = await SourceDefinitionManifestRepository().get(connection, "note", "1") + await connection.execute( + update(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + manifest_fingerprint=manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + evidence = await repository.evidence(connection, "scope", (stored.ref,)) + assert evidence[0].declaration == MemoryEvidenceDeclaration() + assert not any(record.levelno >= logging.WARNING for record in _neutral_records(caplog)) + if legacy_binding: + assert [getattr(record, "reason", None) for record in _neutral_records(caplog)] == ["schema_mismatch"] + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 0adb253bde..6a9abe288a 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -73,8 +73,18 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_001" - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_003" + assert plan.revisions == ( + "s1400", + "s1401", + "s1402", + "v1_4_1_004", + "v1_4_1_005", + "v1_4_1_006", + "v1_5_0_001", + "v1_5_0_002", + "v1_5_0_003", + ) assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" diff --git a/tests/builtin/persistence/test_mysql_initialization_preflight.py b/tests/builtin/persistence/test_mysql_initialization_preflight.py index 2a28e0a8c2..085a49aef0 100644 --- a/tests/builtin/persistence/test_mysql_initialization_preflight.py +++ b/tests/builtin/persistence/test_mysql_initialization_preflight.py @@ -163,3 +163,143 @@ def execute_revision(connection, _config, _source, target, guard): assert connection.exec_driver_sql("SELECT id FROM unregistered").all() == [(7,)] finally: engine.dispose() + + +@pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) +@pytest.mark.parametrize("created_tables", [1, 2, 3]) +def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypatch, backend, created_tables): + from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner + from tests.builtin.persistence.test_atomic_evidence_upgrade_history import ( + configuration as evidence_configuration, + ) + from tests.builtin.persistence.test_atomic_evidence_upgrade_history import ( + previous_upstream_bundle, + ) + + configuration = evidence_configuration(None) + previous = previous_upstream_bundle(tmp_path / "previous-bundle", None) + assert previous.head == "v1_4_1_004" + predecessor = "v1_5_0_001" + database = tmp_path / "managed.db" + sqlite = SQLiteMigrationRunner(database, previous, configuration=configuration) + sqlite.apply(plan_id=sqlite.plan().plan_id, accepted=True, maintenance_confirmed=True) + directory = tmp_path / "native-evidence" + config = ( + SeekDBConfig(path=tmp_path / "native") + if backend == "seekdb" + else OceanBaseConfig(url=SecretStr("mysql+aoceanbase://root@localhost:2881/test?charset=utf8mb4")) + ) + connections = MaintenanceConnections( + config, evidence_directory=directory, lock_coordination="single-host" if backend == "oceanbase" else None + ) + runner = migration.MySQLMigrationRunner( + connections, server_bundle(), evidence_directory=directory, configuration=configuration + ) + identity = BackendIdentity(backend, "fixture-evidence-recovery", "test") + marker = "pc_atomic_memory_evidence_schema" + history = "pc_atomic_memory_evidence_history" + bindings = "pc_source_evidence_bindings" + inventory = copy.deepcopy( + next( + state + for state in runner.schema._prefixes["v1_5_0_002"] + if len({marker, history, bindings} & state["tables"].keys()) == created_tables + ) + ) + control = "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" + inventory["tables"]["pc_schema_revision"] = _parse_table(control)[1] + journal = migration._Journal(directory, identity) + journal.write( + migration._Evidence( + state="active", + database_id=identity.database_id, + source_revision=None, + source_fingerprint=runner.schema.fingerprint({"tables": {}, "views": [], "triggers": []}), + bundle_checksum=runner.checksum, + configuration_digest=runner.configuration_digest, + last_revision=predecessor, + pending_revision="v1_5_0_002", + maintenance_window_id="interrupted-evidence-ddl", + backup_policy="auto", + backup_state="not_required", + release_checksums={ + revision: runner.checksums[revision] + for revision in runner.bundle.revisions[: runner.bundle.revisions.index(predecessor) + 1] + }, + ) + ) + engine = create_engine(f"sqlite:///{database}") + try: + with engine.connect() as connection: + # The interrupted evidence DDL starts only after the released + # description and plugin migrations, including their real version advancement. + config = runner.bundle.config(connection) + config.attributes["configuration"] = configuration + command.upgrade(config, predecessor) + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == predecessor + statements = json.loads((runner.bundle.snapshot_directory / "schemas/sqlite-v1_5_0_002.json").read_text())[ + "create" + ] + for statement in statements[:created_tables]: + connection.exec_driver_sql(statement) + connection.commit() + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == predecessor + monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) + monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) + monkeypatch.setattr( + connections, "run", lambda operation, **_options: operation(connection, identity, lambda: None) + ) + monkeypatch.setattr(connections, "coordination", lambda _identity: {}) + before = database.read_bytes(), journal.path.read_bytes() + plan = runner.plan() + assert plan.state == "recovery_required" and plan.source_revision == predecessor + assert plan.revisions == ("v1_5_0_002", "v1_5_0_003") + assert (database.read_bytes(), journal.path.read_bytes()) == before + assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [] + + def execute_revision(connection, _config, _source, target, guard): + # Model the remaining native DDL, retaining the real domain work, + # version advancement, and original durable recovery evidence. + config = runner.bundle.config(connection) + config.attributes["configuration"] = runner.configuration + if target == "v1_5_0_002": + for statement in statements[created_tables:]: + connection.exec_driver_sql(statement) + command.stamp(config, target) + else: + command.upgrade(config, target) + runner.bundle.domains.apply( + connection, target, backend=backend, configuration=runner.configuration, guard=guard + ) + inventory.clear() + inventory.update(copy.deepcopy(runner.schema._schemas[target])) + inventory["tables"]["pc_schema_revision"] = _parse_table(control)[1] + connection.commit() + + monkeypatch.setattr(runner, "_upgrade", execute_revision) + result = runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert result.state == "ready" and result.revision == "v1_5_0_003" + assert result.backup_state == "not_required" + assert runner.verify().state == "ready" + assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [ + ("p0005",) + ] + completed = journal.read() + assert completed is not None and completed.state == "complete" + assert completed.maintenance_window_id == "interrupted-evidence-ddl" + assert completed.source_revision is None and completed.backup_state == "not_required" + assert completed.last_revision == "v1_5_0_003" and completed.pending_revision is None + assert completed.release_checksums == runner.checksums + assert not runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True).changed + + # An empty marker is resumable only while the original evidence + # records pending DDL, never a repair authorization at completed head. + connection.exec_driver_sql("DELETE FROM pc_atomic_memory_evidence_schema") + connection.commit() + before = database.read_bytes(), journal.path.read_bytes() + for inspect in (runner.plan, runner.verify, runner.apply): + with pytest.raises(MigrationError, match="domain_validation_failed"): + inspect() + assert (database.read_bytes(), journal.path.read_bytes()) == before + finally: + engine.dispose() diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 764990728f..b5e27d1ec1 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -350,10 +350,10 @@ def test_mysql_initialization_and_shared_evidence_noop(backend: _BackendTarget) assert result.state == "ready" assert result.changed assert result.backup_state == "not_required" - assert result.revision == "p0004" + assert result.revision == "p0005" replacement = backend.runner() assert replacement.plan().state == "ready" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert not replacement.apply().changed @@ -373,7 +373,7 @@ def test_mysql_operator_backup_choices_preserve_baseline_data_without_fork( result = _apply(runner, policy=policy) assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") state = _read_state(backend) - assert state["revision"] == "p0004" + assert state["revision"] == "p0005" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert "memory_citations" in state["columns"]["pc_artifacts"] @@ -706,7 +706,7 @@ def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( ) assert runner.plan(backup_policy="manual").source_revision == "s1300" result = _apply(runner, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_003" assert runner.verify().state == "ready" prepare_server_storage(settings) assert not _apply(runner, policy="manual").changed @@ -837,7 +837,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_003", "dedicated", b'{"history":"unchanged"}') (main_path,) = backend.evidence_directory.glob("*/maintenance.json") evidence = json.loads(main_path.read_text()) @@ -847,7 +847,7 @@ def inspect_history(connection, identity, guard): for inspect in (runner.plan, runner.verify, runner.apply): with pytest.raises(MigrationError, match="checksum_conflict"): inspect() - assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_003", "dedicated", b'{"history":"unchanged"}') assert { path: path.read_bytes() for path in backend.evidence_directory.rglob("maintenance.json") } == original_evidence @@ -914,7 +914,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_001", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_003", b'{"history":"unchanged"}') def test_mysql_registered_projection_repair_preserves_authority_and_revision(backend: _BackendTarget) -> None: @@ -982,7 +982,7 @@ def inspect_content(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_content) == ("v1_5_0_001", content, "Preserve immutable nebula evidence.") + assert backend.connections().run(inspect_content) == ("v1_5_0_003", content, "Preserve immutable nebula evidence.") @pytest.mark.parametrize("origin", ("empty", "released")) @@ -1126,8 +1126,17 @@ def load(connection: Connection | None, _identity: BackendIdentity, _guard: Call ("ALTER TABLE pc_artifact_candidate_versions RENAME TO pc_candidate_versions", "s1300"), ("UPDATE pc_artifact_processing_schema SET config_manifest", "s1300"), ("CREATE TABLE pc_memory_artifact_archive", "s1400"), + ("CREATE TABLE pc_atomic_memory_evidence_schema", "v1_5_0_001"), + ("CREATE TABLE pc_source_evidence_bindings", "v1_5_0_001"), ], - ids=("retained-copy", "candidate-rename", "domain-processing", "next-revision"), + ids=( + "retained-copy", + "candidate-rename", + "domain-processing", + "next-revision", + "evidence-marker", + "evidence-bindings", + ), ) def test_mysql_full_server_interruption_preserves_history_and_original_maintenance( backend: _BackendTarget, interruption: str, recorded_revision: str @@ -1137,7 +1146,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan _load_full_server_baseline(backend) runner = _full_server_runner(backend) original = runner.plan(backup_policy="manual") - assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_001" + assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_003" _interrupt_after_ddl(runner, interruption) journals = list(backend.evidence_directory.rglob("maintenance.json")) assert len(journals) == 1 @@ -1159,7 +1168,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan result = _apply(resumed, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_003" assert result.backup_state == "user_confirmed" assert result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" @@ -1239,7 +1248,7 @@ def client_active(connection: Connection | None, _identity: BackendIdentity, _gu # interruption. An arbitrary remote failure cannot be acknowledged blindly. resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_003" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journals[0].read_text(encoding="utf-8")) @@ -1275,7 +1284,7 @@ def test_mysql_committed_partial_ddl_resumes_from_shared_original_evidence(backe replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" state = _read_state(backend) - assert state["revision"] == "p0004" + assert state["revision"] == "p0005" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert replacement.verify().state == "ready" @@ -1321,7 +1330,7 @@ def test_mysql_empty_source_recovers_baseline_and_citation_ddl_without_probing_b ) result = _apply(replacement) assert result.backup_state == "not_required" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert _read_state(backend)["content"] == [] @@ -1338,7 +1347,7 @@ def test_mysql_baseline_adoption_recovers_an_empty_committed_version_table(backe assert recovered.adopt_baseline replacement = _acknowledge_known_completed_interruption(backend, recovered) assert _apply(replacement, policy="manual").state == "ready" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert _read_state(backend)["tags"] == [("Keep",)] @@ -1377,7 +1386,7 @@ def observe_recovery(connection, config, source, target, guard) -> None: patch.setattr(replacement, "_upgrade", observe_recovery) assert _apply(replacement, policy="manual").state == "ready" assert recovered_p0003, "Recovery did not reach p0003" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" state = _read_state(backend) assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] @@ -1388,7 +1397,7 @@ def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledge pytest.skip("embedded seekdb closes its engine and has no remote in-flight run receipt") _load_baseline(backend) assert _apply(backend.runner(), policy="manual").state == "ready" - assert backend.runner().verify().revision == "p0004" + assert backend.runner().verify().revision == "p0005" def committed_write_then_interrupt( connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None] @@ -1402,12 +1411,12 @@ def committed_write_then_interrupt( with pytest.raises(_InterruptedDDL): backend.connections().run(committed_write_then_interrupt, writable=True) before = _read_state(backend) - assert before["revision"] == "p0004" + assert before["revision"] == "p0005" assert before["tags"] == [("Confirmed",)] replacement = backend.runner(backup_factory=_forbidden_automatic_backup) recovery = replacement.plan() assert recovery.state == "recovery_required" - assert recovery.source_revision == "p0004" + assert recovery.source_revision == "p0005" assert recovery.revisions == () pending = (recovery.coordination or {}).get("pending_run_id") assert pending @@ -1426,7 +1435,7 @@ def committed_write_then_interrupt( assert result.state == "ready" assert not result.changed assert _read_state(backend) == before - assert backend.runner().verify().revision == "p0004" + assert backend.runner().verify().revision == "p0005" assert not backend.runner().apply().changed assert "pending_run_id" not in (backend.runner().plan().coordination or {}) diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index 6947e2cb36..5d039d27be 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -129,7 +129,7 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: (directory / "versions/fixture_next.py").write_text( "from alembic import op\n" "revision = 'fixture_next'\n" - "down_revision = 'p0004'\n" + "down_revision = 'p0005'\n" f"def upgrade():\n op.execute({_NEXT_TABLE!r})\n", encoding="utf-8", ) @@ -137,7 +137,7 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: (directory / "mysql_wrappers/fixture_next.py").write_text( "from alembic import context\n" "revision = 'fixture_next'\n" - "down_revision = 'p0004'\n" + "down_revision = 'p0005'\n" "def upgrade():\n context.config.attributes['execute_revision'](revision)\n", encoding="utf-8", ) @@ -172,21 +172,20 @@ def test_registered_next_revision_uses_the_same_recognition_and_sqlite_executor( schema = MySQLSchema(extended_bundle) layouts = json.loads((extended_bundle.snapshot_directory / "mysql_layouts.json").read_text()) tables = _show_tables(citations=2, drop_family_check=True) - for revision in ("p0004",): + for revision in ("p0004", "p0005"): for statement in layouts["revisions"][revision]: tables[statement.split()[2]] = statement source = _ReflectedDatabase(tables) inventory = _inventory(source) - assert schema.identify(inventory) == "p0004" - recovery = schema.recovery(inventory, "p0004") - assert recovery.revision == "fixture_next" + assert schema.identify(inventory) == "p0005" + recovery = schema.recovery(inventory, "p0005") assert recovery.pending_statements == (_NEXT_TABLE,) source.tables["pc_extension_marker"] = _NEXT_TABLE inventory = _inventory(source) assert schema.identify(inventory) == "fixture_next" schema.verify(inventory, "fixture_next") - assert schema.recovery(inventory, "p0004").completed_revision == "fixture_next" - assert extended_bundle.pending("p0003") == ("p0004", "fixture_next") + assert schema.recovery(inventory, "p0005").completed_revision == "fixture_next" + assert extended_bundle.pending("p0004") == ("p0005", "fixture_next") source.tables["pc_schema_revision"] = ( "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" ) @@ -235,11 +234,11 @@ def test_mysql_execution_registration_rejects_incomplete_history_before_opening_ del manifest["mysql_wrappers"] elif change == "predecessor": wrapper = extended_bundle.directory / "mysql_wrappers/fixture_next.py" - wrapper.write_text(wrapper.read_text().replace("'p0004'", "'p0003'"), encoding="utf-8") + wrapper.write_text(wrapper.read_text().replace("'p0005'", "'p0004'"), encoding="utf-8") elif change == "unbound": manifest["revision_resources"]["fixture_next"].remove("mysql_wrappers/fixture_next.py") else: - manifest["mysql_wrappers"]["p0004"] = "mysql_wrappers/fixture_next.py" + manifest["mysql_wrappers"]["p0005"] = "mysql_wrappers/fixture_next.py" path.write_text(json.dumps(manifest), encoding="utf-8") target = tmp_path / "absent-seekdb" evidence = tmp_path / "absent-evidence" diff --git a/tests/builtin/persistence/test_owner_recovery_migration.py b/tests/builtin/persistence/test_owner_recovery_migration.py index de62a3fa0c..c95e4961d1 100644 --- a/tests/builtin/persistence/test_owner_recovery_migration.py +++ b/tests/builtin/persistence/test_owner_recovery_migration.py @@ -15,20 +15,18 @@ """Append owner recovery storage without changing historical migration evidence.""" import json -import shutil import sqlite3 from contextlib import closing import pytest from powercontext.builtin.artifacts.memory import EmbeddingProfile -from powercontext.builtin.persistence.migrations.bundle import MigrationBundle from powercontext.builtin.persistence.migrations.deployment import server_bundle from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase -from tests.builtin.persistence.test_portable_storage_migration import apply, configuration +from tests.builtin.persistence.test_portable_storage_migration import apply, configuration, previous_bundle @pytest.mark.parametrize("vectors", [False, True]) @@ -38,34 +36,7 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks ): embedding = EmbeddingProfile(profile_id="owner-vectors", model="fixture", dimension=3) if vectors else None current = server_bundle(embedding_profile=embedding) - directory = tmp_path / "previous" - shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) - manifest_path = directory / "manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - removed_revisions = ( - ("v1_5_0_001",) - if previous_revision == "v1_4_1_006" - else ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") - if previous_revision == "v1_4_1_004" - else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") - ) - for revision in removed_revisions: - for resource in manifest["revision_resources"].pop(revision): - (directory / resource).unlink() - for script in (directory / "versions").glob(f"{revision}_*.py"): - script.unlink() - (directory / "mysql/versions" / f"{revision}.py").unlink() - del manifest["sqlite_fingerprints"][revision] - del manifest["domain_readiness"][revision] - manifest["mysql_state_layouts"].remove(f"schemas/mysql-{revision}.json") - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in removed_revisions] - (directory / "artifact_storage_readiness.py").unlink() - if previous_revision != "v1_4_1_006": - manifest["affected_objects"].remove("pc_source_write_plugins") - if previous_revision == "s1402": - manifest["affected_objects"].remove("pc_access_owner_recovery") - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - previous = MigrationBundle(directory, embedding_profile=embedding) + previous = previous_bundle(tmp_path / "previous", embedding, through=previous_revision) database = tmp_path / "managed.db" old = SQLiteMigrationRunner(database, previous, configuration=configuration(embedding)) assert apply(old).revision == previous_revision @@ -84,8 +55,9 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration(embedding)) plan = upgraded.plan() - assert plan.source_revision == previous_revision and plan.revisions == removed_revisions - assert apply(upgraded).revision == "v1_5_0_001" + appended = current.revisions[current.revisions.index(previous_revision) + 1 :] + assert plan.source_revision == previous_revision and plan.revisions == appended + assert apply(upgraded).revision == current.head assert upgraded.verify().state == "ready" assert not apply(upgraded).changed evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) @@ -108,8 +80,9 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks assert upgraded.plan().schema_fingerprint == fresh.plan().schema_fingerprint +@pytest.mark.parametrize("target", ["owner", "current"]) @pytest.mark.parametrize("damage", [None, "payload-collation", "pending-index"]) -def test_owner_recovery_mysql_registration_matches_native_character_and_key_definitions(damage): +def test_owner_recovery_mysql_registration_matches_native_character_and_key_definitions(damage, target): # Simulated SHOW output checks normalization, not execution on a native backend. definition = """CREATE TABLE `pc_access_owner_recovery` ( `operation_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, @@ -121,8 +94,9 @@ def test_owner_recovery_mysql_registration_matches_native_character_and_key_defi UNIQUE KEY `uq_pc_access_owner_recovery_pending` (`deployment_id`,`completed`,`operation_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci""" bundle = server_bundle() + target_revision = "v1_4_1_004" if target == "owner" else bundle.head inventory = {"tables": {}, "views": [], "triggers": []} - for revision in bundle.revisions[: bundle.revisions.index("v1_4_1_004")]: + for revision in bundle.revisions[: bundle.revisions.index(target_revision) + 1]: path = bundle.directory / f"schemas/mysql-{revision}.json" fragment = json.loads(path.read_text(encoding="utf-8")) for change in fragment["revisions"][revision]["changes"]: @@ -141,8 +115,8 @@ def test_owner_recovery_mysql_registration_matches_native_character_and_key_defi table["indexes"] = {} schema = MySQLSchema(bundle) if damage is None: - schema.verify(inventory, "v1_4_1_004") - assert schema.identify(inventory) == "v1_4_1_004" + schema.verify(inventory, target_revision) + assert schema.identify(inventory) == target_revision else: with pytest.raises(MigrationError, match="incompatible_schema"): - schema.verify(inventory, "v1_4_1_004") + schema.verify(inventory, target_revision) diff --git a/tests/builtin/persistence/test_portable_storage_migration.py b/tests/builtin/persistence/test_portable_storage_migration.py index 3c286ca286..b083790f77 100644 --- a/tests/builtin/persistence/test_portable_storage_migration.py +++ b/tests/builtin/persistence/test_portable_storage_migration.py @@ -15,6 +15,7 @@ """Managed upgrades retain existing evidence while admitting current portable storage.""" import json +import re import shutil import sqlite3 from contextlib import closing @@ -30,46 +31,50 @@ from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest -def previous_bundle(directory, embedding, *, include_portable=False, include_owner=False): +def _created_tables(path): + """Tables a frozen SQLite schema resource registers, in its exact bytes.""" + + resource = json.loads(path.read_text(encoding="utf-8")) + if "layout" in resource: + return {row[1] for row in resource["layout"] if row[0] == "table"} + payload = resource["create"] if "create" in resource else resource.get("statements", resource["statement"]) + statements = [payload] if isinstance(payload, str) else payload + return { + match[1] + for statement in statements + if (match := re.match(r"CREATE TABLE (?:IF NOT EXISTS )?[`\"]?(\w+)", statement)) + } + + +def previous_bundle(directory, embedding, *, through="s1401"): current = server_bundle(embedding_profile=embedding) shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text()) - later_revisions = [ - ("v1_5_0_001", "artifact_storage", None), - ("v1_4_1_006", "source_write_plugins", "pc_source_write_plugins"), - ("v1_4_1_005", "atomic_memory_description", None), - ] - if not include_owner: - later_revisions.append(("v1_4_1_004", "owner_recovery", "pc_access_owner_recovery")) - for later, suffix, table in later_revisions: - for resource in ( - *manifest["revision_resources"].pop(later), - f"versions/{later}_{suffix}.py", - f"mysql/versions/{later}.py", - ): - (directory / resource).unlink() - del manifest["sqlite_fingerprints"][later] - del manifest["domain_readiness"][later] - manifest["mysql_state_layouts"].remove(f"schemas/mysql-{later}.json") - if table is not None: - manifest["affected_objects"].remove(table) - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != "v1_5_0_001"] - (directory / "artifact_storage_readiness.py").unlink() - if not include_portable: - removed = manifest["revision_resources"].pop("s1402") - for resource in ( - *removed, - "versions/s1402_portable_storage.py", - "mysql/versions/s1402.py", - "portable_storage.py", - ): - (directory / resource).unlink() - del manifest["sqlite_fingerprints"]["s1402"] - del manifest["domain_readiness"]["s1402"] - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != "s1402"] - manifest["mysql_state_layouts"].remove("schemas/mysql-s1402.json") - manifest["affected_objects"].remove("pc_portable_restore_receipts") + revisions = list(manifest["revision_resources"]) + # Prune every revision after the requested previous head so the fixture + # keeps describing the same earlier registration as later revisions are appended. + created = set() + pruned_modules = set() + for revision in revisions[revisions.index(through) + 1 :]: + pruned_modules |= {step["module"] for step in manifest["domain_steps"] if step["revision"] == revision} + for resource in manifest["revision_resources"].pop(revision): + path = directory / resource + if resource.startswith("schemas/sqlite") and resource.endswith(".json"): + created |= _created_tables(path) + path.unlink(missing_ok=True) + if resource in manifest["mysql_state_layouts"]: + manifest["mysql_state_layouts"].remove(resource) + for version in (directory / "versions").glob(f"{revision}_*.py"): + version.unlink() + (directory / "mysql" / "versions" / f"{revision}.py").unlink(missing_ok=True) + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest.get("mysql_wrappers", {}).pop(revision, None) + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != revision] + manifest["affected_objects"] = [name for name in manifest["affected_objects"] if name not in created] + for module in pruned_modules - {step["module"] for step in manifest["domain_steps"]}: + (directory / module).unlink(missing_ok=True) manifest_path.write_text(json.dumps(manifest), encoding="utf-8") return MigrationBundle(directory, embedding_profile=embedding) @@ -113,8 +118,16 @@ def test_registered_s1401_upgrade_preserves_completed_evidence_and_matches_fresh upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) plan = upgraded.plan() assert plan.source_revision == "s1401" - assert plan.revisions == ("s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") - assert apply(upgraded).revision == "v1_5_0_001" + assert plan.revisions == ( + "s1402", + "v1_4_1_004", + "v1_4_1_005", + "v1_4_1_006", + "v1_5_0_001", + "v1_5_0_002", + "v1_5_0_003", + ) + assert apply(upgraded).revision == "v1_5_0_003" assert upgraded.verify().state == "ready" assert not apply(upgraded).changed fresh = SQLiteMigrationRunner(tmp_path / "fresh.db", bundle, configuration=config) @@ -161,7 +174,7 @@ def test_pending_portable_restore_blocks_migration_readiness_without_repair(tmp_ def test_source_write_plugin_upgrade_preserves_data_and_registration(tmp_path, vectors, include_owner): embedding = EmbeddingProfile(profile_id="source-vectors", model="fixture", dimension=3) if vectors else None config = configuration(embedding) - previous = previous_bundle(tmp_path / "previous", embedding, include_portable=True, include_owner=include_owner) + previous = previous_bundle(tmp_path / "previous", embedding, through=("v1_4_1_004" if include_owner else "s1402")) database = tmp_path / "existing.db" old = SQLiteMigrationRunner(database, previous, configuration=config) assert apply(old).revision == ("v1_4_1_004" if include_owner else "s1402") @@ -173,11 +186,11 @@ def test_source_write_plugin_upgrade_preserves_data_and_registration(tmp_path, v bundle = server_bundle(embedding_profile=embedding) upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) assert upgraded.plan().revisions == ( - ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + ("v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") if include_owner - else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + else ("v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001", "v1_5_0_002", "v1_5_0_003") ) - assert apply(upgraded).revision == "v1_5_0_001" + assert apply(upgraded).revision == "v1_5_0_003" definition = b'{"configuration":{"id":"tickets","entry":"tickets"}}' with closing(sqlite3.connect(database)) as connection, connection: assert connection.execute("SELECT title FROM pc_scopes").fetchall() == [("preserved",)] diff --git a/tests/builtin/persistence/test_release140_server_migrations.py b/tests/builtin/persistence/test_release140_server_migrations.py index db8ee5c9f5..ae499d8836 100644 --- a/tests/builtin/persistence/test_release140_server_migrations.py +++ b/tests/builtin/persistence/test_release140_server_migrations.py @@ -134,7 +134,17 @@ def test_registered_sources_choose_distinct_lineages(tmp_path, release, expected _settings, _embedding, runner = _source(tmp_path, release=release) plan = runner.plan() assert plan.source_revision == expected - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_4_1_006", "v1_5_0_001") + assert plan.revisions == ( + "s1400", + "s1401", + "s1402", + "v1_4_1_004", + "v1_4_1_005", + "v1_4_1_006", + "v1_5_0_001", + "v1_5_0_002", + "v1_5_0_003", + ) @pytest.mark.parametrize(("vectors", "code"), [(False, False), (True, False), (False, True), (True, True)]) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 107fa4efaa..0e900a7778 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -15,16 +15,25 @@ from __future__ import annotations import asyncio +import json +import sqlite3 +from typing import cast +from urllib.parse import quote, unquote +import aiosqlite import pytest -from sqlalchemy import insert +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator +from pydantic.alias_generators import to_camel +from sqlalchemy import delete, insert, select, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import dump_model from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.sources import SourceRepository @@ -34,9 +43,13 @@ SHARED_TABLES, SOURCES_TABLE, ) +from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts -from powercontext.builtin.sources import CONTENT_SOURCE_ADAPTER, ContentCapture +from powercontext.builtin.sources import CONTENT_SOURCE_ADAPTER, ContentCapture, ContentSource from powercontext.sources import ( + AdapterSourceDefinition, + MemoryEvidenceDeclaration, + Source, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -52,6 +65,49 @@ ) +class SerializationAliasInput(BaseModel): + source_id: str + body: str + + +class SerializationAliasSource(Source): + body: str = Field(serialization_alias="wire_text") + + +class CamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class UpperCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) + + +class NativeCamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel) + + body: str + + +class SerializationAliasAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + + def __init__(self, source_class: type[SerializationAliasSource]) -> None: + self.source_class = source_class + + async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: + return self.source_class( + name=value.source_id, + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Remote alias description.", + body=value.body, + ) + + async def read(self, source: SerializationAliasSource, /) -> str: + return source.body + + def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -105,6 +161,127 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_source_batch_binds_each_new_identity_and_preserves_replayed_evidence() -> None: + class BatchSource(SerializationAliasSource): + body: str + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(BatchSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + first = await registry.resolve(SerializationAliasInput(source_id="first", body="First retained text.")) + second = await registry.resolve(SerializationAliasInput(source_id="second", body="Second retained text.")) + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + async with profile.database.transaction() as connection: + results = await repository.add_many_with_status(connection, "scope", (first, first, second)) + assert [created for _, created in results] == [True, False, True] + assert [stored.journal_position for stored, _ in results] == [1, 1, 2] + refs = (results[0][0].ref, results[2][0].ref) + evidence = await repository.historical_evidence(connection, "scope", refs) + assert [item.declaration for item in evidence] == [declaration, declaration] + assert await repository.journal_position(connection, "scope") == 2 + + neutral = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(BatchSource), version="2"), + )) + ) + async with profile.database.transaction() as connection: + replayed = await neutral.add_many_with_status(connection, "scope", (second, first, second)) + assert [created for _, created in replayed] == [False, False, False] + assert [stored.journal_position for stored, _ in replayed] == [2, 1, 2] + assert await neutral.historical_evidence(connection, "scope", refs) == evidence + assert await neutral.journal_position(connection, "scope") == 2 + + asyncio.run(scenario()) + + +def test_evidence_reads_large_source_batches_with_sqlite_variable_limit() -> None: + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((AdapterSourceDefinition(SOURCE_ADAPTERS[1], memory_evidence=declaration),)) + repository = SourceRepository(registry) + sources = tuple( + NoteSource(name=f"note-{index}", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + for index in range(501) + ) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add_many_with_status(connection, "scope", sources) + refs = tuple(item.ref for item, _ in stored) + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + previous_limit = await driver._execute(driver._conn.setlimit, sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999) + try: + evidence = await repository.evidence(connection, "scope", (*reversed(refs), refs[0])) + finally: + await driver._execute(driver._conn.setlimit, sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, previous_limit) + assert tuple(item.source_ref for item in evidence) == tuple(reversed(refs)) + assert all(item.declaration == declaration for item in evidence) + + asyncio.run(scenario()) + + +def test_source_batches_register_evidence_again_after_savepoint_rollback() -> None: + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + repository = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SOURCE_ADAPTERS[1], memory_evidence=declaration),)) + ) + sources = tuple( + NoteSource(name=f"note-{index}", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + for index in range(2) + ) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + async with connection.begin_nested() as savepoint: + await repository.add_many_with_status(connection, "scope", sources) + await savepoint.rollback() + stored = await repository.add_many_with_status(connection, "scope", sources) + assert all(created for _, created in stored) + evidence = await repository.evidence(connection, "scope", tuple(item.ref for item, _ in stored)) + assert all(item.declaration == declaration for item in evidence) + + asyncio.run(scenario()) + + +def test_source_batch_validates_each_capture_against_reused_manifest() -> None: + class MixedSource(SerializationAliasSource): + body: str + + @field_serializer("body") + def serialize_body(self, value: str): + return {"text": value} if value == "incompatible" else value + + @field_validator("body", mode="before") + @classmethod + def deserialize_body(cls, value): + return value["text"] if isinstance(value, dict) else value + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(MixedSource), version="2", memory_evidence=declaration), + )) + sources = [ + await registry.resolve(SerializationAliasInput(source_id=f"note-{index}", body=body)) + for index, body in enumerate(("compatible", "incompatible", "compatible")) + ] + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add_many_with_status(connection, "scope", sources) + evidence = await repository.evidence(connection, "scope", tuple(item.ref for item, _ in stored)) + assert [item.declaration for item in evidence] == [declaration, MemoryEvidenceDeclaration(), declaration] + + def test_source_batches_notify_matching_artifact_plugins_only_for_new_inputs() -> None: async def scenario() -> None: intents = ArtifactProcessingIntentRepository() @@ -226,6 +403,473 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("source_class", (SerializationAliasSource, CamelCaseSource, UpperCaseSource)) +def test_remote_observation_ingestion_accepts_serialization_aliases( + source_class: type[SerializationAliasSource], +) -> None: + async def scenario() -> None: + worker = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(source_class), version="2"), + )) + source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) + manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) + observation = project_source_for_transport(worker, source) + + async with repository_profile() as (profile, _repositories): + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + context = await contexts.get("scope-a") + stored = await context.sources.get(observation) + + assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") + assert isinstance(stored, SourceObservation) + assert stored.payload == { + "name": "remote-1", + "definition_version": "2", + "materialization": "captured", + "description": "Remote alias description.", + "body": "Remote note text.", + } + + asyncio.run(scenario()) + + +def test_native_generated_alias_source_survives_read_replay_and_restart(tmp_path) -> None: + async def scenario() -> None: + adapter = SerializationAliasAdapter(NativeCamelCaseSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + value = SerializationAliasInput(source_id="native-2", body="Keep the native version and body.") + source = await adapter.resolve(value) + # Keep the persisted version nondefault regardless of the constructor's alias rules. + source.definition_version = "2" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'native-alias.db'}") + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + assert await context.sources.add(source) == source + loaded = await context.sources.get(source) + assert loaded == source + assert await context.sources.read(loaded) == value.body + assert await context.sources.add(source) == source + assert await context.sources.journal.position(source) == 1 + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + loaded = await context.sources.get(source) + assert loaded == source + assert loaded.definition_version == "2" + resolved = await context.sources.resolve(value) + assert resolved == source + assert await context.sources.add(resolved) == source + assert await context.sources.journal.position(resolved) == 1 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("current_version", ["1", "2"]) +def test_canonical_history_survives_current_identity_aliases(current_version) -> None: + class CurrentSource(SerializationAliasSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + source = SerializationAliasSource( + name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + current = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version=current_version), + )) + ) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "1" + assert expected[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("has_header", [False, True]) +def test_business_name_alias_does_not_reinterpret_canonical_history(has_header: bool) -> None: + class CurrentSource(SerializationAliasSource): + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + current = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="1"),)) + ) + source = SerializationAliasSource( + name="canonical-business-alias", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + material = source.model_dump(mode="json", by_alias=True) + value = {"encoding": "powercontext-source-v1", "representation": "native", "value": material} + if has_header: + value.update( + source=Source( + name=source.name, + definition_version=source.definition_version, + materialization=source.materialization, + ).model_dump(mode="json", exclude_none=True), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE) + .where(SOURCES_TABLE.c.source_id == source.name) + .values(payload=json.dumps(value).encode()) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("business_version", ["1", "2"]) +def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: + class AliasedSource(NativeCamelCaseSource): + definition_version: str = Field(alias="source_version") + materialization: SourceMaterialization = Field(alias="source_materialization") + business_version: str = Field(alias="definition_version") + business_kind: str = Field(alias="materialization") + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(AliasedSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + source = AliasedSource( + name="aliased-history", + source_version="2", + source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, + materialization="referenced", + body="Retained.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": source.model_dump(mode="json", by_alias=True), + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + assert await repository.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "2" + assert expected[0].materialization is SourceMaterialization.CAPTURED + assert expected[0].declaration == declaration + + asyncio.run(scenario()) + + +def test_unbound_versioned_canonical_history_ignores_new_definition_aliases() -> None: + class CurrentSource(SerializationAliasSource): + model_config = ConfigDict(populate_by_name=True) + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + old_source = SerializationAliasSource( + name="old-version-two", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained without an evidence binding.", + ) + old_repository = SourceRepository(()) + + class CurrentAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = CurrentSource + + async def resolve(self, value: SerializationAliasInput, /) -> CurrentSource: + raise AssertionError # Historical evidence must not invoke the current decoder. + + async def read(self, source: CurrentSource, /) -> str: + return source.body + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="3"),)) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + # Model a canonical Source accepted before evidence bindings existed. + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=old_source.name, + payload=json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": old_source.model_dump(mode="json", by_alias=False), + }).encode(), + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=old_source.name) + baseline = await old_repository.historical_evidence(connection, "scope-a", (ref,)) + upgraded = await SourceRepository(registry).historical_evidence(connection, "scope-a", (ref,)) + assert upgraded == baseline + assert upgraded[0].definition_version == "2" + assert upgraded[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: + async def scenario() -> None: + repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) + source = ContentSource( + name="receipt-digest", + materialization=SourceMaterialization.CAPTURED, + content="Receipt content.", + handoff_receipt=True, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + original = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + restored = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + assert restored == original + + asyncio.run(scenario()) + + +def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(NativeCamelCaseSource), version="2"), + )) + repository = SourceRepository(registry) + source = NativeCamelCaseSource( + name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with ( + repository_profile() as (profile, _repositories), + profile.database.transaction() as connection, + ): + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=source.name, + payload=payload, + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(InvalidStoredPayloadError): + await SourceRepository(()).historical_evidence(connection, "scope-a", (ref,)) + loaded = await repository.get(connection, "scope-a", ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == loaded + assert await repository.journal_position(connection, "scope-a") == 1 + + asyncio.run(scenario()) + + +def test_excluded_native_identity_is_rejected_before_capture() -> None: + class ExcludedVersionSource(NativeCamelCaseSource): + definition_version: str = Field(default="1", exclude=True) + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(ExcludedVersionSource),)) + source = ExcludedVersionSource( + name="excluded", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("identity_field", ["name", "definition_version", "materialization"]) +def test_encoded_native_identity_is_rejected_before_capture(identity_field) -> None: + class EncodedSource(NativeCamelCaseSource): + @field_serializer("name", "definition_version", "materialization") + def encode_identity(self, value, info): + if info.field_name == identity_field: + return quote(f"wire:{value}", safe="") + return value + + @field_validator("name", "definition_version", "materialization", mode="before") + @classmethod + def decode_identity(cls, value, info): + if info.field_name == identity_field: + return unquote(value).removeprefix("wire:") + return value + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(EncodedSource),)) + source = EncodedSource(name="note/42", materialization=SourceMaterialization.CAPTURED, body="Retained.") + assert EncodedSource.model_validate_json(source.model_dump_json(by_alias=True)) == source + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + +def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: + class AliasedSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + business_name: str = Field(alias="name") + description: str | None = Field(default="captured description", exclude=True) + + class UpdatedSource(AliasedSource): + description: str | None = Field(default="current description", exclude=True) + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition( + SerializationAliasAdapter(AliasedSource), + version="2", + memory_evidence=declaration, + ), + )) + source = AliasedSource( + source_name="native-alias", + name="business label", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'explicit-alias.db'}") + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add(connection, "scope-a", source) + assert (await repository.get(connection, "scope-a", stored.ref)).value == source + assert await repository.add(connection, "scope-a", source) == stored + evidence = await repository.evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].declaration == declaration + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + updated = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(UpdatedSource), version="2"), + )) + loaded = await SourceRepository(updated).get(connection, "scope-a", stored.ref) + assert loaded.value.description == "current description" + assert await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) == evidence + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("representation", ["enveloped", "bare"]) +def test_replay_retains_historical_native_identity_spelling(representation) -> None: + from pydantic import AliasChoices + + class OldSource(NativeCamelCaseSource): + name: str = Field(alias="legacy_name") + + class CurrentSource(NativeCamelCaseSource): + name: str = Field( + validation_alias=AliasChoices("source_name", "legacy_name"), + serialization_alias="source_name", + ) + + async def scenario() -> None: + original_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(OldSource), version="2"), + )) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="2"), + )) + original = SourceRepository(original_registry) + current = SourceRepository(current_registry) + source = OldSource( + legacy_name="legacy-replay", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained exactly.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + material = source.model_dump(mode="json", by_alias=True) + if representation == "bare": + payload = json.dumps(material).encode() + else: + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + loaded = await current.get(connection, "scope-a", stored.ref) + assert loaded.value.name == source.name + replay = await current.add(connection, "scope-a", loaded.value) + assert replay.ref == stored.ref + assert await current.get(connection, "scope-a", stored.ref) == loaded + evidence = await current.historical_evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].source_ref == stored.ref + raw = await connection.scalar( + select(SOURCES_TABLE.c.payload).where(SOURCES_TABLE.c.source_id == source.name) + ) + assert isinstance(raw, bytes) + envelope = json.loads(raw) + assert envelope["source_keys"]["name"] == "legacy_name" + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -388,3 +1032,45 @@ async def add(scope_id: str, source: NoteSource): assert following.journal_position == 2 asyncio.run(scenario()) + + +def test_nested_uppercase_source_aliases_survive_read_replay_and_restart(tmp_path) -> None: + class UpperNested(Source): + model_config = ConfigDict(alias_generator=str.upper) + body: str + + class NestedNativeSource(NativeCamelCaseSource): + nested: UpperNested + + async def scenario() -> None: + adapter = SerializationAliasAdapter(NestedNativeSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + repository = SourceRepository(registry) + source = NestedNativeSource( + name="nested-legacy", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Outer.", + nested=UpperNested( + name="inner", + materialization=SourceMaterialization.REFERENCED, + body="Nested.", + ), + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nested-alias.db'}") + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + original = await repository.add(connection, "scope-a", source) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + loaded = await repository.get(connection, "scope-a", original.ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == original + evidence = await repository.historical_evidence(connection, "scope-a", (original.ref,)) + assert evidence[0].source_ref == original.ref + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_description_upgrade.py b/tests/e2e/test_atomic_memory_description_upgrade.py index 4387ae9ae7..bebf24b2b8 100644 --- a/tests/e2e/test_atomic_memory_description_upgrade.py +++ b/tests/e2e/test_atomic_memory_description_upgrade.py @@ -85,9 +85,9 @@ def test_nonempty_released_memory_upgrades_without_rewriting_history(tmp_path, r ) result = _apply(runner) - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_003" verified = runner.verify() - assert verified.state == "ready" and verified.readiness_scope == "server" + assert verified.state == "ready" and verified.readiness_scope == "server" and verified.revision == "v1_5_0_003" assert not _apply(runner).changed with closing(sqlite3.connect(runner.database)) as connection: assert ( diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py new file mode 100644 index 0000000000..b03dbb6924 --- /dev/null +++ b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py @@ -0,0 +1,133 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accepted Source declarations do not alter recall or consumer revision citations.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import httpx +import pytest + +from powercontext.builtin.artifacts.handoff import PrepareHandoff +from powercontext.builtin.runtime import ( + BuiltinConfig, + HandoffArtifactCitation, + PrepareContextRequest, + PreparedContext, + RuntimeConfig, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.server.app import ServerApplication, create_app +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.test_atomic_memory_source_evidence import _commit +from tests.e2e.test_handoff_runtime import _EchoHandoffPipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +def test_source_declarations_preserve_recall_and_consumers(migration_backend: MigrationBackend) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=(), atomic_memory_related_mode="fts"), + ) + arms = [] + neutral = MemoryEvidenceDeclaration() + candidates = ( + (SourceRef(source_type="note", source_id="note"), "memory-a", "alpha acceptance"), + (SourceRef(source_type="content", source_id="source"), "memory-b", "alpha guidance"), + ) + for declaration in ( + neutral, + MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ): + registry = SourceDefinitionRegistry(( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration), + AdapterSourceDefinition(NoteAdapter()), + )) + async with open_builtin_runtime( + config, + scheduler_path=migration_backend.directory / "consumer-scheduler.db", + source_registry=registry, + handoff_pipeline=_EchoHandoffPipeline(), + ) as runtime: + assert runtime.scopes is not None and runtime.atomic_memory is not None + scope = await runtime.scopes.create( + ScopeDraft( + title="Recall", + summary="Evidence neutrality.", + idempotency_key=declaration.authority, + ) + ) + scope_id = scope.scope_id + async with open_builtin_contexts(config, source_registry=registry) as contexts: + await contexts.records.capture_source(scope_id, "content", "source", "alpha guidance", {}) + context = await contexts.get(scope_id) + note = await context.sources.resolve(NoteInput(note_id="note", body="alpha acceptance")) + await context.sources.add(note) + memories = [] + for source, artifact_id, text in candidates: + memory = await _commit(contexts, scope_id, artifact_id, text, sources=(source,)) + annotation = await contexts.atomic_memory.for_scope(scope_id).evidence(artifact_id, revision=1) + assert annotation.artifact_ref == memory.ref + expected = declaration if source.source_type == "content" else neutral + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (source, expected), + ) + memories.append(memory) + page = await runtime.atomic_memory.for_scope(scope_id).search("alpha", mode="text", limit=2) + assert tuple((hit.hit.artifact_ref, hit.text, hit.matched_by) for hit in page.hits) == tuple( + (memory.ref, text, ("text",)) for memory, (_, _, text) in zip(memories, candidates, strict=True) + ) + recalled = tuple((hit.hit.artifact_ref.model_dump(mode="json"), hit.text) for hit in page.hits) + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + response = await client.post("/v1/memory/search", json={"scope_id": scope_id, "query": "alpha"}) + assert response.status_code == 200, response.text + public = response.json() + assert ( + tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled + ) + prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) + assert isinstance(prepared, PreparedContext) + assert prepared.status == "ready" and prepared.content is not None + items = json.loads(prepared.content.splitlines()[-2])["items"] + assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled + citation = HandoffArtifactCitation(artifact_ref=page.hits[-1].hit.artifact_ref) + handoffs = runtime.handoff.for_scope(scope_id) + finalized = await handoffs.finalize( + await handoffs.prepare( + PrepareHandoff(objective="Keep the recalled revision.", evidence=(citation,)) + ) + ) + assert finalized.content.state[0].citations == (citation,) + assert finalized.content.next_action is not None + assert finalized.content.next_action.citations == (citation,) + arms.append((page.hits, public, prepared.content.replace(scope_id, "scope"), finalized.content)) + assert arms[0] == arms[1] + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 3075699659..5263ff1a0c 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +from dataclasses import replace from pathlib import Path import pytest @@ -29,6 +30,9 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import SourceDefinitionRegistry +from powercontext.sources.evidence import MemoryEvidenceDeclaration from tests.e2e.atomic_memory_migration_fixture import SCOPE, _atomic, _seed_and_migrate from tests.legacy_memory import add_legacy_citation_columns @@ -36,6 +40,12 @@ @pytest.mark.parametrize("retention", ["complete", "archive_only", "entries_only", "removed", "unreadable"]) def test_rebuild_uses_current_authority_independently_of_legacy_retention(tmp_path: Path, retention: str) -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'projection.db'}") + strong_registry = SourceDefinitionRegistry(( + replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ), + )) async def scenario() -> None: await _seed_and_migrate(config) @@ -43,6 +53,8 @@ async def scenario() -> None: memory = contexts.atomic_memory.for_scope(SCOPE) current = await memory.get(_atomic("alpha", 2).artifact_id) historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) + annotation = await memory.evidence(historical.ref.artifact_id, revision=1) + assert all(source.declaration == MemoryEvidenceDeclaration() for source in annotation.sources) async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: if retention in {"entries_only", "removed"}: @@ -63,10 +75,13 @@ async def scenario() -> None: ) assert report.ready, report.errors assert report.counts["rebuilt_rows"] == 2 - async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + assert report.counts["historical_revisions"] == 3 + assert report.counts["neutral_sources"] == 4 + async with open_builtin_contexts(BuiltinConfig(database=config), source_registry=strong_registry) as contexts: memory = contexts.atomic_memory.for_scope(SCOPE) assert await memory.get(current.ref.artifact_id) == current assert await memory.get(historical.ref.artifact_id, revision=1) == historical + assert await memory.evidence(historical.ref.artifact_id, revision=1) == annotation result = await memory.search("Tasks A and C", mode="text") assert current.ref in tuple(hit.hit.artifact_ref for hit in result.hits) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py new file mode 100644 index 0000000000..fdf2688cd4 --- /dev/null +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -0,0 +1,502 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned source evidence acceptance on disposable SQLite and OceanBase.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import pytest +from pydantic import Field, field_serializer, field_validator +from sqlalchemy import event, text, update +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import ArtifactLineage +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.extraction import ( + AtomicMemoryCandidate, + AtomicMemoryExtractionOutput, +) +from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.persistence import InvalidStoredPayloadError +from powercontext.builtin.persistence.tables import SOURCES_TABLE +from powercontext.builtin.runtime import ( + BuiltinConfig, + SubmitSourceObservation, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.runtime.atomic_memory_rebuild import ( + rebuild_atomic_memory_current, + rebuild_atomic_memory_projection, + verify_atomic_memory_current, +) +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceMaterialization, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from powercontext.sources.observations import manifest_for_definition, project_source_for_transport +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.dream_support import atomic_memory_pipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), artifacts=(), expected_revision=None): + application = contexts.atomic_memory + async with contexts.database.transaction() as connection: + plan = await application.service.inspect_change( + connection, + scope_id, + artifact_id, + AtomicMemoryContent(kind="fact", text=text_value), + None, + expected_revision=expected_revision, + lineage=ArtifactLineage(sources=sources, artifacts=artifacts), + ) + prepared = await application.service.prepare_change(plan) + async with contexts.database.transaction() as connection: + result = await application.service.commit(connection, prepared, None) + return result.primary + + +@pytest.mark.parametrize("declared", [False, True]) +def test_unverifiable_native_serializer_keeps_memory_evidence_neutral(migration_backend, declared) -> None: + class SerializedNote(NoteSource): + count: int = 12 + + @field_serializer("count") + def serialize_count(self, value): + return str(value) + + @field_validator("count", mode="before") + @classmethod + def parse_count(cls, value): + return int(value) + + class SerializedAdapter(NoteAdapter): + source_class = SerializedNote + + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializedAdapter(), memory_evidence=declaration if declared else None), + )) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario(): + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = SerializedNote(name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained.") + await context.sources.add(source) + assert await context.sources.get(source) == source + assert await context.sources.add(source) == source + ref = SourceRef(source_type="note", source_id=source.name) + record = await _commit(contexts, "scope", "serialized", "Retained fact.", sources=(ref,)) + evidence = await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) + assert evidence.artifact_ref == record.ref + assert evidence.sources[0].source_ref == ref + assert evidence.sources[0].declaration == MemoryEvidenceDeclaration() + assert evidence.sources[0].manifest_fingerprint is None + if declared: + # Retain the binding shape accepted before schema compatibility checks. + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "UPDATE pc_source_evidence_bindings SET manifest_fingerprint = " + "(SELECT fingerprint FROM pc_source_definition_manifests WHERE definition_name = 'note'), " + "declaration = :declaration WHERE scope_id = 'scope' AND source_id = 'serialized'" + ), + {"declaration": declaration.model_dump_json().encode()}, + ) + await _commit(contexts, "scope", "legacy", "Previously accepted fact.", sources=(ref,)) + legacy = await contexts.atomic_memory.for_scope("scope").evidence("legacy", revision=1) + assert legacy.sources == evidence.sources + async with open_builtin_contexts(config, source_registry=SourceDefinitionRegistry(())) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + assert await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) == evidence + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("has_header", [False, True]) +@pytest.mark.parametrize("historical_version", ["1", "2"]) +def test_canonical_historical_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool, historical_version: str +) -> None: + class CurrentNote(NoteSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + class CurrentAdapter(NoteAdapter): + source_class = CurrentNote + + config = BuiltinConfig(database=migration_backend.config) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), version=historical_version),)) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(CurrentAdapter(), version=str(int(historical_version) + 1)), + )) + + async def scenario(): + source = NoteSource( + name=f"canonical-v{historical_version}", + definition_version=historical_version, + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + ref = SourceRef(source_type="note", source_id=source.name) + async with open_builtin_contexts(config, source_registry=old_registry) as contexts: + context = await contexts.get("scope") + await context.sources.add(source) + await _commit(contexts, "scope", "canonical", "Canonical historical fact.", sources=(ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload_value = { + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + } + if has_header: + payload_value.update( + source=source.model_dump( + mode="json", + by_alias=False, + include={"name", "definition_version", "materialization"}, + ), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + payload = json.dumps(payload_value).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + + async with open_builtin_contexts(config, source_registry=current_registry) as contexts: + before = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert before.sources[0].source_ref == ref + assert before.sources[0].declaration == MemoryEvidenceDeclaration() + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + after = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert after == before + + asyncio.run(scenario()) + + +def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: + strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") + remote = MemoryEvidenceDeclaration(authority="system_attested", verification="verified") + + class RemoteNoteAdapter(NoteAdapter): + name = "remote-note" + + worker = SourceDefinitionRegistry((AdapterSourceDefinition(RemoteNoteAdapter(), memory_evidence=remote),)) + manifest = manifest_for_definition(worker.definition_for_name("remote-note")) + registry = SourceDefinitionRegistry( + ( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=strong), + AdapterSourceDefinition(NoteAdapter(), memory_evidence=asserted), + ), + remote_declarations={(manifest.name, manifest.version, manifest.fingerprint): remote}, + ) + config = BuiltinConfig(database=migration_backend.config) + refs = ( + SourceRef(source_type="content", source_id="cafe\u0301"), + SourceRef(source_type="note", source_id="native"), + SourceRef(source_type="remote-note", source_id="remote"), + ) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + await contexts.records.capture_source( + "scope", "content", "cafe\u0301", "Release received.", {"authority": "untrusted"} + ) + native = await context.sources.resolve(NoteInput(note_id="native", body="User correction.")) + await context.sources.add(native) + source = await worker.resolve(NoteInput(note_id="remote", body="Remote release evidence.")) + await contexts.register_source_definition(manifest) + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope", observation=project_source_for_transport(worker, source)) + ) + original = await _commit(contexts, "scope", "rule", "Release rule one.", sources=refs) + memory = contexts.atomic_memory.for_scope("scope") + annotation = await memory.evidence("rule", revision=1) + assert annotation.artifact_ref == original.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (refs[0], strong), + (refs[1], asserted), + (refs[2], remote), + ) + async with contexts.database.transaction() as connection: + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, "scope", original.ref) + == annotation + ) + + def reject_snapshot(connection, cursor, statement, *args): + if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): + raise SQLAlchemyError(statement) + + engine = contexts.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", reject_snapshot) + try: + with pytest.raises(SQLAlchemyError): + await _commit(contexts, "scope", "rule", "Failed revision.", expected_revision=1) + finally: + event.remove(engine, "before_cursor_execute", reject_snapshot) + assert await memory.get("rule") == original + revised = await _commit(contexts, "scope", "rule", "Rule two.", sources=(refs[2],), expected_revision=1) + current = await memory.evidence("rule", revision=2) + assert current.sources == (annotation.sources[2],) + neutral_registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION, AdapterSourceDefinition(NoteAdapter()))) + async with open_builtin_contexts(config, source_registry=neutral_registry) as contexts: + await contexts.records.capture_source( + "other", refs[0].source_type, refs[0].source_id, "Other Scope release.", {} + ) + other = await _commit(contexts, "other", "rule", "Other Scope rule.", sources=(refs[0],)) + other_annotation = await contexts.atomic_memory.for_scope("other").evidence("rule", revision=1) + assert other_annotation.scope_id == "other" + assert tuple((item.source_ref, item.declaration) for item in other_annotation.sources) == ( + (refs[0], MemoryEvidenceDeclaration()), + ) + assert other_annotation.sources[0].material_digest != annotation.sources[0].material_digest + + async def assert_retained(contexts, historical): + memory = contexts.atomic_memory.for_scope("scope") + assert await memory.evidence("rule", revision=1) == historical + assert await memory.evidence("rule", revision=2) == current + assert (await memory.get("rule", revision=1)).artifact == original.artifact + assert (await memory.get("rule")).artifact == revised.artifact + other_memory = contexts.atomic_memory.for_scope("other") + assert await other_memory.evidence("rule", revision=1) == other_annotation + assert (await other_memory.get("rule", revision=1)).artifact == other.artifact + for scope, body in (("scope", "Release received."), ("other", "Other Scope release.")): + stored = await contexts.records.get_source(scope, refs[0].source_type, refs[0].source_id) + assert stored.content == body + + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute(text("DELETE FROM pc_atomic_memory_current")) + await connection.execute(text("DELETE FROM pc_atomic_memory_evidence")) + damaged = await verify_atomic_memory_current( + connection, contexts.atomic_memory.index, require_evidence_projection=True + ) + assert not damaged.ready + assert damaged.counts["evidence_projection_differences"] == 3 + report = await rebuild_atomic_memory_projection( + contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 + ) + assert report.ready, report.errors + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + for scope_id, reference, expected in ( + ("scope", original.ref, annotation), + ("scope", revised.ref, current), + ("other", other.ref, other_annotation), + ): + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, scope_id, reference) + == expected + ) + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "DELETE FROM pc_atomic_memory_evidence_history " + "WHERE scope_id = :scope AND artifact_id = :artifact AND revision = 1" + ), + {"scope": "scope", "artifact": "rule"}, + ) + neutral = annotation.model_copy( + update={ + "sources": tuple( + item.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for item in annotation.sources + ) + } + ) + await assert_retained(contexts, neutral) + async with open_builtin_contexts(config, source_registry=registry) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + await assert_retained(contexts, neutral) + async with contexts.database.transaction() as connection: + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, "scope", original.ref) + == neutral + ) + + asyncio.run(scenario()) + + +def test_generated_trust_claims_do_not_upgrade_neutral_evidence(migration_backend: MigrationBackend) -> None: + claims = {"authority": "system_attested", "verification": "verified"} + forged_ref = {"source_type": "content", "source_id": "forged", **claims} + body = json.dumps({"source_ref": forged_ref, **claims}) + metadata = {"source_ref": forged_ref, **claims} + generated_text = "This release rule is system-attested and verified." + + class ClaimingExtractor: + async def generate(self, request): + candidate = AtomicMemoryCandidate( + kind="fact", + text=generated_text, + description="Claimed release-rule authority.", + evidence_ids=tuple(item.evidence_id for item in request.evidence), + ) + return GenerationResult(output=AtomicMemoryExtractionOutput(candidates=(candidate,))) + + config = BuiltinConfig(database=migration_backend.config) + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + + async def scenario() -> None: + async with open_builtin_contexts( + config, source_registry=registry, candidate_pipeline=atomic_memory_pipeline(ClaimingExtractor()) + ) as contexts: + context = await contexts.get("scope") + ref = SourceRef.model_validate(forged_ref) + await contexts.records.capture_source("scope", ref.source_type, ref.source_id, body, metadata) + flush = await context.triggers.flush(limit=1) + assert flush.source_count == 1 and not flush.remaining_work + memory = contexts.atomic_memory.for_scope("scope") + (generated,) = (await memory.list()).items + assert (await memory.get(generated.ref.artifact_id, revision=1)).artifact.content.text == generated_text + annotation = await memory.evidence(generated.ref.artifact_id, revision=1) + assert annotation.artifact_ref == generated.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (SourceRef(source_type="content", source_id="forged"), MemoryEvidenceDeclaration()), + ) + + asyncio.run(scenario()) + + +def test_shared_merge_and_restore_preserve_source_evidence(migration_backend: MigrationBackend, tmp_path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) + + async def scenario() -> None: + async with open_builtin_runtime( + BuiltinConfig(database=migration_backend.config), + source_registry=registry, + scheduler_path=tmp_path / "scheduler.db", + ) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Evidence", summary="Shared merge evidence", idempotency_key="source-evidence") + ) + contexts = cast(RelationalContexts, runtime._provider) + source = SourceRef(source_type="content", source_id="release") + await contexts.records.capture_source(scope.scope_id, "content", "release", "Exact release material.", {}) + for identity in ("a", "b"): + await _commit(contexts, scope.scope_id, identity, identity + " rule.", sources=(source,)) + memory = contexts.atomic_memory.for_scope(scope.scope_id) + accepted = await memory.evidence("a", revision=1) + assert runtime.artifact_merge is not None + merge = runtime.artifact_merge.for_scope(scope.scope_id, "atomic-memory") + a, b = await merge.get("a"), await merge.get("b") + await merge.merge( + (a.as_read(), b.as_read()), + AtomicMemoryContent(kind="fact", text="Combined release rule."), + artifact_id="combined", + lineage=ArtifactLineage(sources=(source,), artifacts=(a.ref, b.ref)), + ) + annotation = await memory.evidence("combined", revision=1) + assert annotation.sources == accepted.sources + assert annotation.sources[0].declaration == declaration + restored = await merge.restore("a") + assert restored.primary.ref.revision == 2 + assert (await memory.evidence("a", revision=2)).sources[0].declaration == declaration + assert await memory.evidence("combined", revision=1) == annotation + assert await memory.evidence("a", revision=1) == accepted + + asyncio.run(scenario()) + + +def test_current_rebuild_accepts_the_configured_source_registry(migration_backend: MigrationBackend) -> None: + class AliasedNote(NoteSource): + # Retained material keys this identity by its business alias, so only the + # registered Definition can decode it; the builtin registry cannot. + name: str = Field(alias="source_name") + + class AliasedAdapter(NoteAdapter): + source_class = AliasedNote + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(AliasedAdapter()),)) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = AliasedNote.model_validate({ + "source_name": "aliased", + "materialization": "captured", + "body": "Retained.", + }) + await context.sources.add(source) + ref = SourceRef(source_type="note", source_id=source.name) + await _commit(contexts, "scope", "rebuild", "Retained fact.", sources=(ref,)) + # Replay a row captured before evidence bindings existed: no identity + # header, only the business-aliased native material. + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == "aliased").values(payload=payload) + ) + memory = contexts.atomic_memory.for_scope("scope") + expected = await memory.evidence("rebuild", revision=1) + + # Maintenance must judge the row exactly as the configured read path does; + # the builtin registry cannot decode it and must not report success. + with pytest.raises(InvalidStoredPayloadError): + async with contexts.database.transaction() as connection: + await rebuild_atomic_memory_current(connection, contexts.atomic_memory.index) + + async with contexts.database.transaction() as connection: + report = await rebuild_atomic_memory_current( + connection, contexts.atomic_memory.index, sources=contexts.repositories.sources + ) + assert report.ready, report.errors + assert await memory.evidence("rebuild", revision=1) == expected + + asyncio.run(scenario()) diff --git a/tests/e2e/test_managed_archive_recovery.py b/tests/e2e/test_managed_archive_recovery.py index 5e8344734d..6589cbe685 100644 --- a/tests/e2e/test_managed_archive_recovery.py +++ b/tests/e2e/test_managed_archive_recovery.py @@ -59,7 +59,7 @@ async def scenario(): ) artifact = await source.records.create_artifact(scope.scope_id, family, ArtifactWrite(content=content)) authoritative = await source.records.get_artifact(scope.scope_id, family, artifact.artifact_id) - await source.portability.export([scope.scope_id], archive, authorize=authorize) + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) async with open_builtin_contexts(config, embedding_model=embedding) as target: with pytest.raises(RuntimeError, match="interrupted projection rebuild"): await PortableBundleService( @@ -86,6 +86,10 @@ async def scenario(): async with open_builtin_contexts(config, embedding_model=embedding) as target: restored = await target.records.get_artifact(scope.scope_id, family, artifact.artifact_id) assert restored.content == authoritative.content + recovered = await target.portability.export( + [scope.scope_id], tmp_path / "recovered.pcb", authorize=authorize + ) + assert recovered.total_digest == exported.total_digest asyncio.run(scenario()) assert ( diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 2691b68079..1197762b5c 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -50,12 +50,16 @@ ) from powercontext.builtin.inference import GenerationResult from powercontext.builtin.inference.pydantic_ai import PydanticAIEmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, AtomicMemorySearchChannels, AtomicMemorySearchRequest, ) -from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + atomic_memory_current_table, +) from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinConfig, CaptureSource, open_builtin_runtime from powercontext.builtin.runtime.artifact_processing import ( @@ -236,14 +240,26 @@ class _VectorMemoryIndex: capabilities = AtomicMemoryIndexCapabilities(fts=False, vector=True, embedding_profile=_VECTOR_PROFILE) table = atomic_memory_current_table() - tables = () + tables = (ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE,) def __init__(self, profile=None): - pass + self._evidence = AtomicMemoryEvidenceRepository() async def initialize(self, connection: AsyncConnection, /): pass + async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read(connection, scope_id, artifact_ref) + + async def project_evidence(self, connection, annotation, /): + await self._evidence.replace_projection(connection, annotation) + + async def read_evidence_projection(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read_projection(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /): pass diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index d01ca7b89a..0c3ae668e1 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -34,7 +34,7 @@ from sqlalchemy import create_engine, event, insert, select, update from sqlalchemy.engine import make_url -from powercontext.artifacts import ArtifactAddress, ArtifactRef, StableHandle +from powercontext.artifacts import ArtifactAddress, ArtifactLineage, ArtifactRef, StableHandle from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent from powercontext.builtin.artifacts.atomic_memory.errors import InvalidAtomicMemoryStateError from powercontext.builtin.artifacts.experience import ExperienceContent @@ -83,6 +83,7 @@ from powercontext.limits import MAX_ARTIFACT_ID_LENGTH from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, SourceDefinitionRegistry, SourceRef, manifest_for_definition, @@ -93,6 +94,61 @@ from tests.e2e.atomic_memory_migration_fixture import _seed_and_migrate +def test_archive_preserves_accepted_native_evidence(tmp_path: Path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), memory_evidence=declaration),)) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "evidence.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(source_config, source_registry=registry) as source: + scope = await source.scopes.create( + ScopeDraft(title="Evidence", summary="Portable evidence", idempotency_key="evidence") + ) + context = await source.get(scope.scope_id) + native = await context.sources.resolve(NoteInput(note_id="native", body="Retained evidence.")) + await context.sources.add(native) + ref = SourceRef(source_type="note", source_id="native") + service = source.atomic_memory.service + async with source.database.transaction() as connection: + plan = await service.inspect_change( + connection, + scope.scope_id, + "rule", + AtomicMemoryContent(kind="fact", text="Preserve evidence."), + None, + lineage=ArtifactLineage(sources=(ref,)), + ) + prepared = await service.prepare_change(plan) + async with source.database.transaction() as connection: + await service.commit(connection, prepared, None) + expected = await source.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert expected.sources[0].declaration == declaration + await source.portability.export([scope.scope_id], archive, authorize=authorize) + inspection = await source.portability.inspect(archive) + assert inspection.records_by_type["source_evidence_binding"] == 1 + assert inspection.records_by_type["atomic_memory_evidence_history"] == 1 + assert inspection.records_by_type["source_definition"] == 1 + async with open_builtin_contexts(target_config, source_registry=registry) as target: + await target.portability.restore(archive) + async with open_builtin_contexts(target_config, source_registry=registry) as target: + actual = await target.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert actual == expected + async with target.database.transaction() as connection: + assert ( + await target.atomic_memory_index.read_evidence_projection( + connection, scope.scope_id, expected.artifact_ref + ) + == expected + ) + + asyncio.run(scenario()) + + def test_topic_restore_can_retry_after_projection_failure_and_restart(tmp_path: Path) -> None: async def authorize(_scopes: tuple[str, ...]) -> None: pass diff --git a/tests/native/test_oceanbase_engine_recovery.py b/tests/native/test_oceanbase_engine_recovery.py index 12b189c72b..0a11c51009 100644 --- a/tests/native/test_oceanbase_engine_recovery.py +++ b/tests/native/test_oceanbase_engine_recovery.py @@ -175,7 +175,7 @@ def ping(connection, _identity, _guard): _apply(_full_server_runner(backend), policy="manual") resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.revision == "v1_5_0_001" and result.readiness_scope == "server" + assert result.state == "ready" and result.revision == "v1_5_0_003" and result.readiness_scope == "server" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journal.read_text()) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 75a2517134..ea2e5f69fb 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -401,7 +401,7 @@ def test_native_public_migration_switches_an_installed_130_service( if failure in {"startup", "switch"}: assert applied.exit_code == 1, applied.output assert result["error"] == "service_start_failed" - assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_001" + assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_003" assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} phase = "switching" if failure == "switch" else "start_failed" assert json.loads(controller.maintenance_path.read_text())["phase"] == phase diff --git a/tests/test_managed_atomic_maintenance.py b/tests/test_managed_atomic_maintenance.py index 41f968cbbd..450c804fba 100644 --- a/tests/test_managed_atomic_maintenance.py +++ b/tests/test_managed_atomic_maintenance.py @@ -76,7 +76,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) assert plan["source_revision"] == "s1300" - assert plan["target_revision"] == "v1_5_0_001" + assert plan["target_revision"] == "v1_5_0_003" assert plan["readiness_scope"] == "server" assert historical.read_bytes() == original @@ -103,7 +103,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b with closing(sqlite3.connect(result["backup_ref"])) as connection: assert tuple(connection.iterdump()) == original_dump with closing(sqlite3.connect(historical)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] assert connection.execute("SELECT content FROM pc_artifacts").fetchall() == [(b'{"history":"unchanged"}',)] verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) assert verified.exit_code == 0, verified.output diff --git a/tests/test_managed_processing_maintenance.py b/tests/test_managed_processing_maintenance.py index b803b5ada9..7377ea3eeb 100644 --- a/tests/test_managed_processing_maintenance.py +++ b/tests/test_managed_processing_maintenance.py @@ -75,7 +75,7 @@ def test_registered_mode_switch_backs_up_preserves_revision_and_fences_old_worke assert result.backup_state == "completed" assert result.backup_ref and Path(result.backup_ref).is_file() with closing(sqlite3.connect(path)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] assert ( json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ "mode" diff --git a/tests/test_migration_revision_names.py b/tests/test_migration_revision_names.py index d620895262..89f7799b42 100644 --- a/tests/test_migration_revision_names.py +++ b/tests/test_migration_revision_names.py @@ -62,9 +62,11 @@ def invoke(action, *options): "v1_4_1_005", "v1_4_1_006", "v1_5_0_001", + "v1_5_0_002", + "v1_5_0_003", ] assert [item["revision"] for item in plan["pending_migrations"]] == list(reviewed.revisions) - target = {"revision": "v1_5_0_001", "name": "v1_5_0_001", "software_version": "1.5.0", "sequence": 1} + target = {"revision": "v1_5_0_003", "name": "v1_5_0_003", "software_version": "1.5.0", "sequence": 3} assert plan["target_migration"] == target applied = invoke( @@ -79,7 +81,7 @@ def invoke(action, *options): ) assert applied["migration"] == target and applied["server_ready"] with closing(sqlite3.connect(runner.database)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_003",)] evidence = runner.evidence_path.read_bytes() recorded = json.loads(evidence) assert recorded["source_revision"] == expected_source["revision"] @@ -92,6 +94,8 @@ def invoke(action, *options): "v1_4_1_005", "v1_4_1_006", "v1_5_0_001", + "v1_5_0_002", + "v1_5_0_003", } assert runner.verify().state == "ready" assert invoke("verify")["migration"] == target diff --git a/tests/test_registered_source_maintenance_cli.py b/tests/test_registered_source_maintenance_cli.py index 2680ee13b4..5809d3781d 100644 --- a/tests/test_registered_source_maintenance_cli.py +++ b/tests/test_registered_source_maintenance_cli.py @@ -66,7 +66,7 @@ def test_release140_compatibility_commands_preserve_source_and_share_full_upgrad planned = CliRunner().invoke(cli, [*arguments, "plan", "--backup", "skip"]) assert planned.exit_code == 0, planned.output plan = json.loads(planned.output) - assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_001" + assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_003" assert plan["readiness_scope"] == "server" assert plan["compatibility_steps"] == [] full = CliRunner().invoke(cli, ["server", "db-migrate", "plan", "--backup", "skip"]) diff --git a/tests/test_server_logging.py b/tests/test_server_logging.py index bc2b6c660c..a236d29420 100644 --- a/tests/test_server_logging.py +++ b/tests/test_server_logging.py @@ -510,6 +510,7 @@ def test_json_formatter_emits_stable_operational_fields() -> None: "verdict": "deny", "final_action": "memory_write_hold", "fallback_reason": "decision_model_fallback", + "reason": "manifest_conflict", "candidate_count": 1, "evidence_count": 2, "ignored": "not serialized", @@ -536,6 +537,7 @@ def test_json_formatter_emits_stable_operational_fields() -> None: assert payload["verdict"] == "deny" assert payload["final_action"] == "memory_write_hold" assert payload["fallback_reason"] == "decision_model_fallback" + assert payload["reason"] == "manifest_conflict" assert (payload["candidate_count"], payload["evidence_count"]) == (1, 2) assert "ignored" not in payload diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index b4933a7934..be10272920 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,20 +15,33 @@ from __future__ import annotations import asyncio +import hashlib import urllib.request +from decimal import Decimal +from typing import Self import pytest -from pydantic import ValidationError +import rfc8785 +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_serializer +from pydantic.alias_generators import to_camel from referencing.exceptions import Unresolvable +from powercontext import InvalidSourceObservationError +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import SHARED_TABLES +from powercontext.builtin.runtime import SubmitSourceObservation +from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture from powercontext.builtin.sources.observation_validation import json_schema_validator from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + AdapterSourceDefinition, Source, SourceCatalog, SourceDefinitionManifest, SourceDefinitionRegistry, + SourceMaterialization, SourceObservation, TextEvidence, manifest_for_definition, @@ -75,6 +88,537 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize( + ("name_alias", "body_alias"), + [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], +) +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion( + name_alias: str, body_alias: str, tmp_path +) -> None: + class AliasedSource(Source): + name: str = Field(serialization_alias=name_alias) + definition_version: str = Field(default="1", serialization_alias="wire_version") + materialization: SourceMaterialization = Field(serialization_alias="wire_materialization") + description: str | None = Field(default=None, serialization_alias="wire_description") + body: str = Field(serialization_alias=body_alias) + + class AliasedAdapter: + name = "aliased" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Aliased description.", + body=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Retain the aliased business value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "remote-aliased", + "definition_version": "2", + "materialization": "captured", + "description": "Aliased description.", + "body": "Retain the aliased business value.", + } + assert manifest.source_schema == AliasedSource.model_json_schema() + baseline_declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": AliasedSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline_declaration)).hexdigest()}" + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload == expected + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-manifest.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + + current_manifest = manifest_for_definition(definition) + assert current_manifest == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(current_manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "remote-aliased" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as native_profile: + async with native_profile.database.transaction() as connection: + native_manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + assert native_manifest is not None + assert native_manifest.source_schema == AliasedSource.model_json_schema(mode="serialization") + assert native_manifest.fingerprint != manifest.fingerprint + + asyncio.run(scenario()) + + +def test_alias_serialization_remote_manifest_and_payload_remain_upgrade_stable(tmp_path) -> None: + class AliasedSource(Source): + model_config = ConfigDict(serialize_by_alias=True, populate_by_name=True) + body: str = Field(alias="wire_text") + + class AliasedAdapter: + name = "aliased-upgrade" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + wire_text=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained alias.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + expected = AliasedSource.model_json_schema() + expected_fingerprint = hashlib.sha256( + rfc8785.dumps({ + "name": definition.name, + "version": definition.version, + "source_schema": expected, + "projections": [], + }) + ).hexdigest() + assert manifest.source_schema == expected + assert manifest.fingerprint == f"sha256:{expected_fingerprint}" + assert observation.payload["wire_text"] == "Retained alias." + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-alias-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "remote-aliased" + + asyncio.run(scenario()) + + +def test_optional_alias_with_forbidden_extras_accepts_the_worker_payload() -> None: + class OptionalAliasSource(Source): + model_config = ConfigDict(extra="forbid") + body: str = Field(default="Default body.", alias="wire_text") + + class OptionalAliasAdapter: + name = "optional-alias" + input_class = str + source_class = OptionalAliasSource + + async def resolve(self, value: str, /) -> OptionalAliasSource: + return OptionalAliasSource.model_validate({ + "name": "optional", + "materialization": SourceMaterialization.CAPTURED, + "body": value, + }) + + async def read(self, source: OptionalAliasSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(OptionalAliasAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the optional value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "optional" + assert isinstance(stored, SourceObservation) + assert stored.payload["body"] == "Retain the optional value." + + asyncio.run(scenario()) + + +def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: + class VersionedSource(Source): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + body: str + + class VersionedAdapter: + name = "versioned-source" + input_class = str + source_class = VersionedSource + + async def resolve(self, value: str, /) -> VersionedSource: + return VersionedSource( + name="versioned", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + ) + + async def read(self, source: VersionedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(VersionedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": VersionedSource.model_json_schema(), + "projections": [], + } + assert manifest.source_schema == VersionedSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["definition_version"] == "2" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'optional-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "versioned" + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("wrapper", ["default", "list"]) +def test_populated_optional_nested_alias_accepts_the_worker_payload(wrapper: str) -> None: + # A field with a concrete default always reaches the payload, so a renamed nested + # shape below it must be described by the serialization schema. A null default is + # covered by test_nullable_nested_alias_upgrade_keeps_the_accepted_manifest. + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True) + label: str = Field(alias="wire_label") + + class DefaultNestedSource(Source): + detail: Detail = Field(default_factory=lambda: Detail.model_validate({"label": "Default label."})) + + class ListNestedSource(Source): + detail: list[Detail] = Field(default_factory=list) + + source_model = { + "default": DefaultNestedSource, + "list": ListNestedSource, + }[wrapper] + + class NestedAliasAdapter: + name = "optional-nested-alias" + input_class = str + source_class = source_model + + async def resolve(self, value: str, /) -> Source: + detail = Detail.model_validate({"label": value}) + return source_model.model_validate({ + "name": "optional-nested", + "materialization": SourceMaterialization.CAPTURED, + "detail": [detail] if wrapper == "list" else detail, + }) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(NestedAliasAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the nested value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + expected_detail = {"label": "Retain the nested value."} + assert receipt.source_ref.source_id == "optional-nested" + assert isinstance(stored, SourceObservation) + assert stored.payload["detail"] == ([expected_detail] if wrapper == "list" else expected_detail) + + asyncio.run(scenario()) + + +def test_nullable_nested_alias_upgrade_keeps_the_accepted_manifest(tmp_path) -> None: + """A payload that can only carry null cannot justify replacing an accepted manifest.""" + + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True) + label: str = Field(alias="wire_label") + + class NullableSource(Source): + detail: Detail | None = None + + class NullableAdapter: + name = "nullable-nested" + input_class = str + source_class = NullableSource + + async def resolve(self, value: str, /) -> NullableSource: + return NullableSource( + name="nullable", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + detail=None, + ) + + async def read(self, source: NullableSource, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(NullableAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": NullableSource.model_json_schema(), + "projections": [], + } + # Renaming the nested key only matters to material this Definition never emits. + assert manifest.source_schema == NullableSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["detail"] is None + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-nested.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "nullable" + # Populating the field needs a representation the accepted manifest describes, + # so the flat field-name payload is rejected instead of silently accepted. + populated = project_source_for_transport( + registry, + NullableSource( + name="nullable", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + detail=Detail.model_validate({"label": "Populated."}), + ), + ) + with pytest.raises(InvalidSourceObservationError, match="schema"): + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=populated) + ) + + asyncio.run(scenario()) + + +def test_nullable_nested_alias_serializing_by_alias_accepts_populated_payloads(tmp_path) -> None: + """A nested model that serializes by alias keeps the accepted manifest and populated values.""" + + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True) + label: str = Field(alias="wire_label") + + class NullableSource(Source): + detail: Detail | None = None + + class PopulatedAdapter: + name = "nullable-nested-alias" + input_class = str + source_class = NullableSource + + async def resolve(self, value: str, /) -> NullableSource: + return NullableSource( + name="populated", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + detail=Detail.model_validate({"label": value}), + ) + + async def read(self, source: NullableSource, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(PopulatedAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the nested value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + assert manifest.source_schema == NullableSource.model_json_schema() + assert observation.payload["detail"] == {"wire_label": "Retain the nested value."} + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "populated" + assert isinstance(stored, SourceObservation) + assert stored.payload["detail"] == {"wire_label": "Retain the nested value."} + + asyncio.run(scenario()) + + +def test_decimal_source_manifest_fingerprint_survives_upgrade(tmp_path) -> None: + class AmountSource(Source): + amount: Decimal + + class AmountAdapter: + name = "amount-source" + input_class = str + source_class = AmountSource + + async def resolve(self, value: str, /) -> AmountSource: + return AmountSource( + name="amount", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + amount=Decimal(value), + ) + + async def read(self, source: AmountSource, /) -> str: + return str(source.amount) + + async def scenario() -> None: + definition = AdapterSourceDefinition(AmountAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("1.25") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": AmountSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert manifest.source_schema == AmountSource.model_json_schema() + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload["amount"] == "1.25" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'decimal-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "amount" + + asyncio.run(scenario()) + + +def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: + class RecursiveSource(Source): + name: str = Field(serialization_alias="source_name") + definition_version: str = Field(default="1", serialization_alias="wire_version") + body: str = Field(serialization_alias="wire_text") + children: list[Self] = Field(default_factory=list) + + class RecursiveAdapter: + name = "recursive" + input_class = str + source_class = RecursiveSource + + async def resolve(self, value: str, /) -> RecursiveSource: + leaf = RecursiveSource(name="leaf", materialization=SourceMaterialization.CAPTURED, body="Nested text.") + return RecursiveSource( + name="root", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + children=[leaf], + ) + + async def read(self, source: RecursiveSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(RecursiveAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Root text.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "root", + "definition_version": "2", + "materialization": "captured", + "description": None, + "body": "Root text.", + "children": [ + { + "name": "leaf", + "definition_version": "1", + "materialization": "captured", + "description": None, + "body": "Nested text.", + "children": [], + } + ], + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "root" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) @@ -87,6 +631,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_identity_exclusions_and_serializers_fail_observation_validation() -> None: + class ExcludedIdentitySource(Source): + definition_version: str = Field(default="1", exclude=True) + + class TransformedIdentitySource(Source): + @field_serializer("name") + def encode_name(self, value: str) -> str: + return f"wire:{value}" + + class Adapter: + input_class = str + name = "remote-identity" + + def __init__(self, source_class: type[Source]) -> None: + self.source_class = source_class + + async def resolve(self, value: str, /) -> Source: + return self.source_class(name=value, materialization=SourceMaterialization.CAPTURED) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + for source_class in (ExcludedIdentitySource, TransformedIdentitySource): + definition = AdapterSourceDefinition(Adapter(source_class), version="1") + registry = SourceDefinitionRegistry((definition,)) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema() + source = await registry.resolve("identity") + with pytest.raises(ValidationError): + project_source_for_transport(registry, source) + + asyncio.run(scenario()) + + def test_remote_schema_references_are_rejected_without_network_access(monkeypatch: pytest.MonkeyPatch) -> None: attempted = False @@ -109,3 +687,65 @@ def reject_request(*_args: object, **_kwargs: object) -> None: fingerprint=f"sha256:{'0' * 64}", source_schema={"$dynamicRef": "https://example.invalid/schema"}, ) + + +def test_covered_union_retains_registered_manifest_and_accepts_both_branches() -> None: + class Aliased(BaseModel): + model_config = ConfigDict(populate_by_name=True, extra="forbid") + label: str = Field(validation_alias="wire_label") + + class Canonical(BaseModel): + model_config = ConfigDict(extra="forbid") + label: str + + class UnionSource(Source): + item: Aliased | Canonical + + class UnionAdapter: + name = "covered-union" + input_class = str + source_class = UnionSource + + async def resolve(self, value: str, /) -> UnionSource: + return UnionSource.model_validate({ + "name": value, + "definition_version": "1", + "materialization": "captured", + "item": {"label": value}, + }) + + async def read(self, source: UnionSource, /) -> str: + return source.item.label + + async def scenario() -> None: + definition = AdapterSourceDefinition(UnionAdapter(), version="1") + declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": UnionSource.model_json_schema(), + "projections": [], + } + historical = SourceDefinitionManifest.model_validate( + declaration + | { + "fingerprint": "sha256:" + hashlib.sha256(rfc8785.dumps(declaration)).hexdigest(), + } + ) + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(historical) + await contexts.register_source_definition(manifest_for_definition(definition)) + for index, branch in enumerate((Aliased.model_validate({"label": "a"}), Canonical(label="b"))): + source = UnionSource.model_validate({ + "name": f"branch-{index}", + "definition_version": "1", + "materialization": "captured", + "item": branch, + }) + observation = project_source_for_transport(SourceDefinitionRegistry((definition,)), source) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == source.name + + asyncio.run(scenario())