From 8439875a32a7de9b54b5a5dc4cfdc0fb85f98a33 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 14:35:27 +0800 Subject: [PATCH 01/26] feat(memory): persist versioned source evidence Implement issue01 of the Source evidence declarations specification. Bind trusted native and remote declarations to exact Source material and retain per-revision Atomic Memory snapshots in the existing transaction. Preserve neutral fallback, exact identities, scope authorization and public retrieval contracts. Add versioned p0004 migrations and SQLite persistence regressions. Refs #1975. Full suite: 5255 passed, 431 skipped, 7 failed; introduced observability adapter failures fixed and its file passes 18 tests. Topic Memory vector failure also reproduces at the unchanged baseline. --- .../powercontext_connector_opendal/source.py | 2 + src/powercontext/__init__.py | 4 + .../artifacts/atomic_memory/service.py | 8 + .../persistence/atomic_memory_evidence.py | 161 ++++++++++++++++++ .../atomic_memory_evidence_schema.py | 59 +++++++ .../persistence/atomic_memory_index.py | 52 +++++- .../migrations/atomic_memory_evidence.py | 100 +++++++++++ .../versions/p0004_atomic_memory_evidence.py | 25 +++ .../persistence/migrations/mysql_schema.py | 6 +- .../migrations/resources/manifest.json | 17 +- .../schemas/atomic_memory_evidence_v1.json | 14 ++ .../versions/p0004_atomic_memory_evidence.py | 30 ++++ .../builtin/persistence/schema.py | 10 +- .../builtin/persistence/source_definitions.py | 35 +++- .../builtin/persistence/sources.py | 158 ++++++++++++++++- .../builtin/runtime/atomic_memory.py | 55 ++++++ .../builtin/runtime/composition.py | 14 +- src/powercontext/sources/__init__.py | 3 + src/powercontext/sources/definitions.py | 63 ++++++- src/powercontext/sources/evidence.py | 60 +++++++ 20 files changed, 860 insertions(+), 16 deletions(-) create mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py create mode 100644 src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py create mode 100644 src/powercontext/sources/evidence.py diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 5ab48cf7d6..c089367cba 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,6 +23,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -92,6 +93,7 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource + memory_evidence: MemoryEvidenceDeclaration | None = None def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..f7589beabc 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -51,12 +51,14 @@ ) from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceAdapter, SourceCatalog, SourceCatalogBackend, SourceDefinition, SourceDefinitionRegistry, + SourceEvidence, SourceMaterialization, SourceProjection, SourceProjectionKey, @@ -90,6 +92,7 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", + "MemoryEvidenceDeclaration", "PolicyTransition", "PowerContext", "PowerContextError", @@ -104,6 +107,7 @@ "SourceDefinitionNotFoundError", "SourceDefinitionRegistry", "SourceError", + "SourceEvidence", "SourceMaterialization", "SourceNotFoundError", "SourceProjection", diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 0947035cd5..90fb984ef4 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -80,6 +80,9 @@ async def publish( prepared: PreparedAtomicMemoryProjection, ) -> None: ... async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: ... + async def capture_evidence( + self, connection: AsyncConnection, scope_id: str, record: AtomicMemoryRecord + ) -> None: ... ATOMIC_MERGE_ERRORS = ArtifactMergeErrors( @@ -478,6 +481,11 @@ async def commit_window( read_set=tuple(_common_read(read) for read in read_set), direct_source=direct_source, ) + for item, result in zip(prepared, results, strict=True): + revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} + for record in result.records: + if record.artifact.artifact_id in revisions: + await self.projections.capture_evidence(connection, scope_id, _atomic_record(record)) return tuple( AtomicMemoryMutationResult( changed=result.changed, diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py new file mode 100644 index 0000000000..cb054ff704 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable per-revision evidence inputs and their neutral internal annotation.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Literal + +import rfc8785 +from aiosqlite import Connection as SQLiteConnection +from pydantic import BaseModel, ConfigDict, ValidationError +from sqlalchemy import insert, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE +from powercontext.builtin.persistence.sources import _require_identity +from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence + + +class AtomicMemoryEvidenceAnnotation(BaseModel): + """Exact revision facts, not a trust score or recursively inferred evidence.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + artifact_ref: ArtifactRef + sources: tuple[SourceEvidence, ...] + format: Literal["powercontext.atomic-memory.evidence.v1"] = "powercontext.atomic-memory.evidence.v1" + + +class AtomicMemoryEvidenceRepository: + """Join caller transactions; retained inputs are never deleted with the index. + + The annotation is derived from retained inputs at read time. There is no + separate cache to confuse with accepted history or join into search. + """ + + async def save( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + ) -> AtomicMemoryEvidenceAnnotation: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is not None: + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + accepted = [] + for source in sources: + value = source.model_dump(mode="json") + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + annotation = AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(accepted) + ) + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + # SQLite legacy transaction mode does not BEGIN on SELECT; + # SAVEPOINT must not accidentally own and commit the write. + await connection.exec_driver_sql("BEGIN") + try: + async with connection.begin_nested(): + await connection.execute( + insert(ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE).values( + scope_id=scope_id, + artifact_id=artifact_ref.artifact_id, + revision=artifact_ref.revision, + sources=rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]), + ) + ) + except IntegrityError: + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is None: + raise + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + return annotation + + async def read( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + @staticmethod + def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + reference = ArtifactRef.model_validate(artifact_ref.model_dump()) + if reference.family != "atomic-memory": + from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError + + raise InvalidRepositoryArgumentError("artifact_ref", "must reference an Atomic Memory revision") + return reference + + @staticmethod + async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef) -> object | None: + table = ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE + return await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + + @staticmethod + def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: + try: + if not isinstance(payload, (bytes, bytearray, memoryview)): + return None + values = json.loads(bytes(payload)) + if not isinstance(values, list): + return None + sources = [] + for value in values: + if not isinstance(value, dict): + return None + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) + except (ValueError, TypeError): + # An unreadable declaration snapshot conveys no historical claim. + # This does not catch SQL errors or actual Source resolution errors. + return None + + @staticmethod + def _neutral( + scope_id: str, artifact_ref: ArtifactRef, sources: Sequence[SourceEvidence] + ) -> AtomicMemoryEvidenceAnnotation: + # Preserve a damaged row exactly, without replacing it with today's + # stronger configuration. A caller can only receive neutral facts. + return AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, + artifact_ref=artifact_ref, + sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), + ) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py new file mode 100644 index 0000000000..b4b446a20a --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retained exact-version evidence, independent of disposable search projections.""" + +from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, Table, Text +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT + +from powercontext.builtin.persistence.tables import SHARED_METADATA, identity_string +from powercontext.limits import ( + MAX_ARTIFACT_ID_LENGTH, + MAX_SCOPE_ID_LENGTH, + MAX_SOURCE_ID_LENGTH, + MAX_SOURCE_TYPE_LENGTH, +) + +ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION = "p0004" +ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE = Table( + "pc_atomic_memory_evidence_schema", + SHARED_METADATA, + Column("schema_version", identity_string(32), primary_key=True), +) +ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE = Table( + "pc_atomic_memory_evidence_history", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_history_revision"), +) +SOURCE_EVIDENCE_BINDINGS_TABLE = Table( + "pc_source_evidence_bindings", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), primary_key=True), + Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), primary_key=True), + Column("material_digest", identity_string(71), nullable=False), + Column("definition_name", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("definition_version", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("manifest_fingerprint", identity_string(71)), + Column("declaration", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), +) +ATOMIC_MEMORY_EVIDENCE_TABLES = ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 4cb2cbb00a..d125a46861 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -18,6 +18,7 @@ import json import math +from collections.abc import Sequence from dataclasses import dataclass, replace from hashlib import sha256 from typing import Any, Literal, Protocol @@ -37,9 +38,16 @@ fts_query_requirements, ) from powercontext.builtin.inference import EmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import SourceEvidence AtomicMemorySearchMode = Literal["fts", "vector", "hybrid"] ATOMIC_MEMORY_RRF_CONSTANT = 60 @@ -166,6 +174,19 @@ async def replace( async def delete(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: ... + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: ... + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -191,9 +212,11 @@ def __init__( index: AtomicMemoryIndex, *, embedding_model: EmbeddingModel | None = None, + sources: SourceRepository | None = None, ) -> None: self.index = index self._embedding_model = embedding_model + self._sources = sources @property def profile_fingerprint(self) -> str | None: @@ -243,6 +266,17 @@ async def prepare(self, content: BaseModel) -> PreparedAtomicMemoryProjection: embedding_input_hash=None if vector is None else atomic_memory_embedding_input_hash(kind, body), ) + async def capture_evidence(self, connection: AsyncConnection, scope_id: str, record: Any) -> None: + """Accept history only for a committed body write, never state-only republishing.""" + + artifact = record.artifact + if ( + self._sources is not None + and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None + ): + evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) + await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + async def publish( self, connection: AsyncConnection, @@ -450,12 +484,28 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table,) + self.tables: tuple[Table, ...] = (table, *ATOMIC_MEMORY_EVIDENCE_TABLES) + self._evidence = AtomicMemoryEvidenceRepository() self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read(connection, scope_id, artifact_ref) + def _encode_embedding(self, vector: tuple[float, ...]) -> object: return vector diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py new file mode 100644 index 0000000000..cafc2cdfe9 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply the frozen additive p0004 revision to runtime-owned evidence tables. + +The production bundle covers a historical subset of business tables. Runtime +profiles therefore apply this same frozen additive revision independently of +that bundle's whole-database inventory, without stamping unrelated schema or +backfilling declarations for old Source/Artifact rows. SQLite DDL joins the +caller's transaction; MySQL DDL may commit and is resumed by exact postconditions. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from aiosqlite import Connection as SQLiteConnection +from sqlalchemy import inspect, select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION, + ATOMIC_MEMORY_EVIDENCE_TABLES, +) +from powercontext.builtin.persistence.errors import PersistenceError + + +class AtomicMemoryEvidenceSchemaError(PersistenceError): + """An evidence schema is not the registered versioned layout.""" + + def __init__(self, detail: str) -> None: + message = { + "version": "Atomic Memory evidence schema version is unsupported", + "shape": "Atomic Memory evidence table has an incompatible shape", + }[detail] + super().__init__(message) + + +async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> None: + """Add only missing p0004 tables, validate exact shapes, then mark ready. + + No reads or writes of Source/Artifact history are needed, and no current + declaration is used to invent accepted historical inputs. + """ + + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + await connection.exec_driver_sql("BEGIN IMMEDIATE") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + exists = await connection.run_sync(lambda sync, name=table.name: inspect(sync).has_table(name)) + if not exists: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + versions = tuple((await connection.execute(select(marker.c.schema_version))).scalars()) + if not versions: + await connection.exec_driver_sql(statements[-1]) + elif versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + + +async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: + if dialect == "sqlite": + actual = await connection.exec_driver_sql( + "SELECT type, name, sql FROM sqlite_schema WHERE (name = ? OR tbl_name = ?) " + "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", + (name, name), + ) + if actual.all() != [("table", name, statement)]: + raise AtomicMemoryEvidenceSchemaError("shape") + return + # Reuse the frozen MySQL inventory parser so physical formatting accepted + # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table + + actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + if not _matches( + {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, + {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, + ): + raise AtomicMemoryEvidenceSchemaError("shape") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py new file mode 100644 index 0000000000..e83df8afbf --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the frozen additive evidence layout without backfilling trust.""" + +from alembic import context, op + +revision = "p0004" +down_revision = "p0003" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) + op.execute("INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index b3502db1bc..01cf62dc69 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -35,7 +35,7 @@ from .models import MigrationError, digest _LEXEME = re.compile(r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+-]") -_REVISIONS = ("p0001", "p0002", "p0003") +_REVISIONS = ("p0001", "p0002", "p0003", "p0004") _CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") _FAMILY_CHECK = "ck_pc_artifact_tags_family" _CONTROL_TABLE = "pc_schema_revision" @@ -141,10 +141,14 @@ def __init__(self, bundle: MigrationBundle) -> None: self.bundle = bundle baseline = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text())["mysql"] current = json.loads((bundle.snapshot_directory / "schemas/v1_1_0.json").read_text())["mysql"] + evidence = json.loads((bundle.snapshot_directory / "schemas/atomic_memory_evidence_v1.json").read_text())[ + "mysql" + ] self._statements = { "p0001": tuple(baseline), "p0002": tuple(f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" for table in _CITATION_TABLES), "p0003": (f"ALTER TABLE pc_artifact_tags DROP CHECK {_FAMILY_CHECK}",), + "p0004": tuple(evidence[:-1]), } inventory = _empty_inventory() self._prefixes: dict[str, list[dict[str, Any]]] = {} diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 65c798ea8b..73dbd2a358 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -4,11 +4,15 @@ "schemas/pre_dream.json" ], "p0002": [], - "p0003": [] + "p0003": [], + "p0004": [ + "schemas/atomic_memory_evidence_v1.json" + ] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", - "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003" + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", + "93c1c20ae9bba2114158658e7852324af2bbaef12d28fa298cdbe3cae4847a84": "p0004" }, "sqlite_fingerprints": { "p0001": [ @@ -20,13 +24,20 @@ "p0003": [ "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" + ], + "p0004": [ + "4f21a735d9ff2e53b70599f658d762ee30a14604e4fe96c97826215433dcd509", + "93c1c20ae9bba2114158658e7852324af2bbaef12d28fa298cdbe3cae4847a84" ] }, "affected_objects": [ "pc_artifacts", "pc_artifact_heads", "pc_artifact_candidate_versions", - "pc_artifact_tags" + "pc_artifact_tags", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_source_evidence_bindings" ], "retained_objects": [ "pc_retained_p0002_artifact_tags" diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json new file mode 100644 index 0000000000..fdd79d655e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json @@ -0,0 +1,14 @@ +{ + "sqlite": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')" + ], + "mysql": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py new file mode 100644 index 0000000000..32648cfd55 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen additive evidence revision; never synthesize historical declarations.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0004" +down_revision = "p0003" + + +def upgrade() -> None: + statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text()) + connection = op.get_bind() + for statement in statements["sqlite" if connection.dialect.name == "sqlite" else "mysql"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index d1e4170bc3..3fff568777 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -21,12 +21,20 @@ from sqlalchemy import MetaData, Table from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration + async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: - """Create missing tables and indexes using SQLAlchemy's native metadata support.""" + """Create selected tables, applying retained evidence through its frozen revision.""" groups: dict[MetaData, list[Table]] = {} + evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): + await apply_atomic_memory_evidence_migration(connection) for table in tables: + if table.name in evidence_names: + continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 9e5d4a8640..3705e9b566 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Persistence for worker-owned Source Definition manifests.""" +"""Immutable historical manifests owned by the Source registration boundary.""" from __future__ import annotations @@ -26,15 +26,17 @@ from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, + InvalidStoredColumnError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinitionManifest +from powercontext.sources import SourceDefinition, SourceDefinitionManifest, manifest_for_definition class SourceDefinitionManifestRepository: - """Register immutable worker-owned Definition manifests by name and version.""" + """Retain native and worker manifests without changing their identities.""" async def register( self, @@ -59,7 +61,9 @@ async def register( ) except IntegrityError: existing = await self.find(connection, manifest.name, manifest.version) - if existing is None or existing != manifest: + if existing is None: + raise + if existing != manifest: raise StoredPayloadConflictError( "source-definition-manifest", (manifest.name, manifest.version), @@ -67,6 +71,29 @@ async def register( return existing return manifest + async def register_native( + self, + connection: AsyncConnection, + definition: SourceDefinition[Any, Any, Any], + /, + ) -> SourceDefinitionManifest | None: + """Retain the existing manifest representation for trusted native material. + + This internal registration cannot replace an older manifest. A missing, + invalid or conflicting ancillary manifest leaves the Source declaration + neutral; database failures still abort the caller's transaction. External + registration keeps its existing active-Definition ownership guard. + """ + + try: + manifest = manifest_for_definition(definition) + except Exception: # Native schema providers may not expose a valid manifest. + return None + try: + return await self.register(connection, manifest) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError, StoredPayloadConflictError): + return None + async def get( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..5b38249f1f 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -18,6 +18,7 @@ from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress +from hashlib import sha256 from typing import Any, Literal, cast import rfc8785 @@ -29,6 +30,7 @@ from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, @@ -39,12 +41,22 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE from powercontext.builtin.sources.content import ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import Source, SourceAdapter, SourceDefinitionRegistry, SourceObservation, SourceRef +from powercontext.sources import ( + MemoryEvidenceDeclaration, + Source, + SourceAdapter, + SourceDefinitionManifest, + SourceDefinitionRegistry, + SourceEvidence, + SourceObservation, + SourceRef, +) _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -165,6 +177,7 @@ async def add_with_status( if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) from None return stored, False + await self._bind_memory_evidence(connection, scope_id, ref, source) # Centralized here so capture, record projection and import all publish # discoverable input atomically. Replayed Source identities do not dirty # a binding again. Disabled processors simply retain ordinary dirty. @@ -233,6 +246,123 @@ async def get_many( raise RepositoryNotFoundError("source", (scope_id, ref)) return tuple(decoded[(ref.source_type, ref.source_id)] for ref in ordered) + async def evidence( + self, + connection: AsyncConnection, + scope_id: str, + refs: Sequence[SourceRef], + /, + ) -> tuple[SourceEvidence, ...]: + """Read accepted declarations against exact stored Source material. + + Source reads retain their existing errors and never become generation + eligibility checks. Missing or invalid declaration bindings are neutral; + database failures propagate. No current registry declaration or provider + value is substituted for historical registration facts. + """ + + stored_sources = await self.get_many(connection, scope_id, refs) + # These server-owned records describe a management write, not evidence + # supplied for that write. Other lineage-only Sources remain represented. + sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) + if not sources: + return () + table = SOURCE_EVIDENCE_BINDINGS_TABLE + rows = ( + ( + await connection.execute( + select(table).where( + table.c.scope_id == scope_id, + tuple_(table.c.source_type, table.c.source_id).in_( + tuple((stored.ref.source_type, stored.ref.source_id) for stored in sources) + ), + ) + ) + ) + .mappings() + .all() + ) + bindings = {(row["source_type"], row["source_id"]): row for row in rows} + manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} + evidence: list[SourceEvidence] = [] + for stored in sources: + source = stored.value + digest = _source_material_digest(source) + value = SourceEvidence( + source_ref=stored.ref, + materialization=source.materialization, + material_digest=digest, + definition_name=stored.ref.source_type, + definition_version=source.definition_version, + ) + binding = bindings.get((stored.ref.source_type, stored.ref.source_id)) + if binding is None or ( + binding["material_digest"] != digest + or binding["definition_name"] != value.definition_name + or binding["definition_version"] != value.definition_version + or not isinstance(binding["manifest_fingerprint"], str) + ): + evidence.append(value) + continue + identity = (value.definition_name, value.definition_version) + if identity not in manifests: + manifests[identity] = await _find_evidence_manifest(connection, *identity) + manifest = manifests[identity] + if ( + manifest is None + or manifest.fingerprint != binding["manifest_fingerprint"] + or (isinstance(source, SourceObservation) and source.definition_fingerprint != manifest.fingerprint) + ): + evidence.append(value) + continue + try: + declaration = MemoryEvidenceDeclaration.model_validate_json( + stored_bytes(binding["declaration"], column="declaration") + ) + except (InvalidStoredColumnError, TypeError, ValueError): + declaration = MemoryEvidenceDeclaration() + evidence.append( + value.model_copy( + update={ + "manifest_fingerprint": manifest.fingerprint, + "declaration": declaration, + } + ) + ) + return tuple(evidence) + + async def _bind_memory_evidence( + self, + connection: AsyncConnection, + scope_id: str, + ref: SourceRef, + source: Source, + ) -> None: + """Freeze registration evidence only for a newly accepted Source row.""" + + if isinstance(source, SourceObservation): + manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint != source.definition_fingerprint: + manifest = None + else: + definition = self._registry.definition_for_source(source) + manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + declaration = ( + MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) + ) + await connection.execute( + insert(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + scope_id=scope_id, + source_type=ref.source_type, + source_id=ref.source_id, + material_digest=_source_material_digest(source), + definition_name=ref.source_type, + definition_version=source.definition_version, + manifest_fingerprint=None if manifest is None else manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + async def list( self, connection: AsyncConnection, @@ -442,6 +572,32 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded +def _source_material_digest(source: Source) -> str: + # Preserve every material field except the narrow server witness upgrade. + material = source.model_dump(mode="json", by_alias=True) + if isinstance(source, ContentSource): + material.pop("handoff_receipt", None) + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _management_write_source(source: Source) -> bool: + return ( + isinstance(source, ContentSource) + and source.internal is not None + and source.internal.role == "lineage_only" + and source.internal.operation in {"artifact_create", "artifact_replace"} + ) + + +async def _find_evidence_manifest( + connection: AsyncConnection, name: str, version: str +) -> SourceDefinitionManifest | None: + try: + return await SourceDefinitionManifestRepository().find(connection, name, version) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + return None + + def _dump_source( source: Source, *, diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index f79cb78fcb..f2a2a14ed9 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -58,6 +58,7 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository, atomic_memory_state_expression +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -79,6 +80,7 @@ from powercontext.builtin.records import BaseOperationNotSupportedError, InvalidBaseAccessRequestError from powercontext.builtin.runtime.artifact_merge import ArtifactMergeAccess, merge_artifact_tags from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import MemoryEvidenceDeclaration class AtomicMemoryAccess(ArtifactMergeAccess): @@ -186,6 +188,7 @@ def __init__( self.publisher = AtomicMemoryProjectionPublisher( index, embedding_model=embedding_model, + sources=artifacts.source_repository, ) self.service = AtomicMemoryService( artifacts=artifacts, @@ -222,6 +225,58 @@ async def get(self, artifact_id: str, *, revision: int | None = None, context=No connection, self.scope_id, artifact_id, context, revision=revision ) + async def evidence(self, artifact_id: str, *, revision: int, context=None) -> AtomicMemoryEvidenceAnnotation: + """Inspect neutral evidence facts for one authorized exact body revision. + + A missing historical snapshot remains unknown; current registration + cannot attest a revision that did not accept a declaration. + """ + application = self.application + async with ( + deferred_decision_audit(context), + application.database.transaction(consistent_snapshot=True) as connection, + ): + record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) + sources = application.artifacts.source_repository + materials = ( + () + if sources is None + else await sources.evidence(connection, self.scope_id, record.artifact.lineage.sources) + ) + annotation = await application.index.read_evidence(connection, self.scope_id, record.ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryIndexError( + "evidence-material", "Historical evidence does not match the exact Source material" + ) + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + return AtomicMemoryEvidenceAnnotation( + scope_id=self.scope_id, + artifact_ref=record.ref, + sources=tuple( + material.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for material in materials + ), + ) + verified = tuple( + accepted + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ) + else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for accepted, material in zip(annotation.sources, materials, strict=True) + ) + return annotation.model_copy(update={"sources": verified}) + async def list( self, *, diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index b8015b896c..cd2e4b9e40 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -196,7 +196,7 @@ ) from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError -from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey +from powercontext.sources import AdapterSourceDefinition, Source, SourceDefinitionRegistry, SourceProjectionKey if TYPE_CHECKING: from pydantic_ai.models import Model @@ -796,7 +796,17 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( - definition is not expected for definition, expected in zip(definitions, builtin_definitions, strict=True) + not ( + definition is expected + or ( + type(definition) is AdapterSourceDefinition + and type(expected) is AdapterSourceDefinition + and definition.adapter is expected.adapter + and definition.version == expected.version + and definition.projections is expected.projections + ) + ) + for definition, expected in zip(definitions, builtin_definitions, strict=True) ) ): raise BuiltinConfigurationError("artifact-processing-source-registry") diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 91e1a27b08..5469244670 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,6 +30,7 @@ SourceDefinitionRegistry, SourceProjection, ) +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef from powercontext.sources.observations import ( SourceDefinitionManifest, @@ -53,6 +54,7 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", + "MemoryEvidenceDeclaration", "Source", "SourceAdapter", "SourceCatalog", @@ -60,6 +62,7 @@ "SourceDefinition", "SourceDefinitionManifest", "SourceDefinitionRegistry", + "SourceEvidence", "SourceMaterialization", "SourceObservation", "SourceProjection", diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d9c54572dc..18f6bd5162 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -18,6 +18,7 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -34,6 +35,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter +from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -42,6 +44,7 @@ _JSON_VALUE = TypeAdapter(JsonValue) _AnySourceAdapter = SourceAdapter[Any, Any, Any] +_NEUTRAL_MEMORY_EVIDENCE = MemoryEvidenceDeclaration() class SourceProjection(Protocol[SourceT]): @@ -60,6 +63,7 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] + memory_evidence: MemoryEvidenceDeclaration | None @dataclass(frozen=True, slots=True) @@ -69,6 +73,7 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () + memory_evidence: MemoryEvidenceDeclaration | None = None @property def input_class(self) -> type[InputT]: @@ -93,14 +98,28 @@ async def read(self, source: SourceT, /) -> ValueT_co: class SourceDefinitionRegistry: - """Provide one immutable routing view for Source persistence and consumers.""" - - def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: + """Provide one immutable routing and trusted declaration view. + + Native Definitions optionally supply ``memory_evidence``. Deployment-owned + ``remote_declarations`` keys are exact ``(name, version, fingerprint)`` + manifest identities, never worker payload or projection values. Declarations + are validated and copied at construction; invalid or unavailable values are + neutral and never prevent registration of an otherwise valid Definition. + """ + + def __init__( + self, + definitions: Iterable[_AnySourceDefinition], + /, + *, + remote_declarations: Mapping[tuple[str, str, str], MemoryEvidenceDeclaration] | None = None, + ) -> None: by_input: dict[type[object], _AnySourceDefinition] = {} by_source: dict[type[Source], _AnySourceDefinition] = {} by_name: dict[str, _AnySourceDefinition] = {} projections: dict[type[Source], Mapping[SourceProjectionKey, SourceProjection[Any]]] = {} registered: list[_AnySourceDefinition] = [] + memory_evidence: dict[type[Source], MemoryEvidenceDeclaration] = {} for definition in definitions: input_class, source_class = _validate_definition(definition) @@ -122,6 +141,11 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: by_source[source_class] = definition by_name[definition.name] = definition projections[source_class] = projection_routes + try: + declaration = getattr(definition, "memory_evidence", None) + except Exception: # Declaration providers may be unavailable; routing remains valid. + declaration = None + memory_evidence[source_class] = _snapshot_memory_evidence(declaration) registered.append(definition) self._definitions = tuple(registered) @@ -129,6 +153,10 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: self._by_source = by_source self._by_name = by_name self._projections = projections + self._memory_evidence = MappingProxyType(memory_evidence) + self._remote_declarations = MappingProxyType({ + key: _snapshot_memory_evidence(value) for key, value in (remote_declarations or {}).items() + }) @classmethod def from_adapters(cls, adapters: Iterable[_AnySourceAdapter], /) -> SourceDefinitionRegistry: @@ -161,6 +189,23 @@ def definition_for_source(self, source: object, /) -> _AnySourceDefinition: ) return definition + def memory_evidence_for_source(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Read the frozen trusted declaration using the exact registered route.""" + + from powercontext.sources.observations import SourceObservation + + if isinstance(source, SourceObservation): + return self.memory_evidence_for_manifest( + source.source_type, source.definition_version, source.definition_fingerprint + ) + self.definition_for_source(source) + return self._memory_evidence[type(source)] + + def memory_evidence_for_manifest(self, name: str, version: str, fingerprint: str, /) -> MemoryEvidenceDeclaration: + """Return only deployment-configured evidence for an exact manifest.""" + + return self._remote_declarations.get((name, version, fingerprint), _NEUTRAL_MEMORY_EVIDENCE) + async def resolve(self, value: object, /) -> Source: input_class = type(value) try: @@ -203,6 +248,18 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error +def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: + if value is None: + return _NEUTRAL_MEMORY_EVIDENCE + try: + # Revalidate even constructed models, and never retain caller-owned data. + if isinstance(value, MemoryEvidenceDeclaration): + value = value.model_dump(mode="python") + return MemoryEvidenceDeclaration.model_validate(value) + except (TypeError, ValueError): + return _NEUTRAL_MEMORY_EVIDENCE + + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py new file mode 100644 index 0000000000..33eebfed9f --- /dev/null +++ b/src/powercontext/sources/evidence.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed declarations and exact material facts accepted at Source registration.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.sources.models import SourceMaterialization, SourceRef + + +class MemoryEvidenceDeclaration(BaseModel): + """A trusted adapter's claim, not a score, permission or verification engine. + + ``declaration_version`` identifies this contract, independently of the + Definition version and Artifact content revision. Missing or unsupported + declarations resolve to ``untrusted`` / ``unknown``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + authority: Literal["untrusted", "user_asserted", "repository_attested", "system_attested"] = "untrusted" + verification: Literal["unknown", "verified", "not_verified"] = "unknown" + declaration_version: Literal["1"] = "1" + + +class SourceEvidence(BaseModel): + """Retain one exact Source material binding without duplicating its body. + + The digest checks equality of canonical Source JSON, excluding only the + server-owned ContentSource handoff receipt witness. It authenticates neither + the Source nor its declaration. Each Source retains its own declaration. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_ref: SourceRef + materialization: SourceMaterialization + material_digest: str + definition_name: str + definition_version: str + manifest_fingerprint: str | None = None + declaration: MemoryEvidenceDeclaration = Field(default_factory=MemoryEvidenceDeclaration) + + +__all__ = ["MemoryEvidenceDeclaration", "SourceEvidence"] From 5dfa2ad62bc8ba1515de43c3181a5c1e3013e54f Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 15:41:42 +0800 Subject: [PATCH 02/26] feat(memory): verify historical evidence during projection rebuild --- .../persistence/atomic_memory_evidence.py | 52 ++++- .../builtin/persistence/sources.py | 196 ++++++++++++++++-- .../builtin/runtime/atomic_memory.py | 50 +---- .../builtin/runtime/atomic_memory_rebuild.py | 58 +++++- 4 files changed, 291 insertions(+), 65 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py index cb054ff704..99d5c38311 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -29,8 +29,9 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE -from powercontext.builtin.persistence.sources import _require_identity +from powercontext.builtin.persistence.sources import SourceRepository, _require_identity from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources import SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence @@ -104,6 +105,55 @@ async def read( payload = await self._payload(connection, scope_id, artifact_ref) return None if payload is None else self._decode(scope_id, artifact_ref, payload) + async def verify( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + source_refs: Sequence[SourceRef], + *, + sources: SourceRepository | None = None, + ) -> AtomicMemoryEvidenceAnnotation: + """Derive exact facts from retained inputs, never today's declarations. + + Missing or damaged declaration records are neutral. Missing Sources, + corrupt authoritative payloads and unequal exact materials remain errors. + No accepted snapshot is saved or repaired by this operation. + """ + + from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError + from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY + + artifact_ref = self._reference(scope_id, artifact_ref) + repository = SourceRepository(BUILTIN_SOURCE_REGISTRY) if sources is None else sources + materials = await repository.historical_evidence(connection, scope_id, source_refs) + annotation = await self.read(connection, scope_id, artifact_ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryIndexError( + "evidence-material", "Historical evidence does not match the exact Source material" + ) + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + return self._neutral(scope_id, artifact_ref, materials) + verified = tuple( + accepted + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ) + else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for accepted, material in zip(annotation.sources, materials, strict=True) + ) + return annotation.model_copy(update={"sources": verified}) + @staticmethod def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 5b38249f1f..597c89a842 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -22,7 +22,12 @@ from typing import Any, Literal, cast import rfc8785 +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from referencing import Registry +from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -43,7 +48,7 @@ from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE -from powercontext.builtin.sources.content import ContentSource +from powercontext.builtin.sources.content import CONTENT_SOURCE_NAME, ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH @@ -267,6 +272,167 @@ async def evidence( sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) if not sources: return () + materials = tuple( + SourceEvidence( + source_ref=stored.ref, + materialization=stored.value.materialization, + material_digest=_source_material_digest(stored.value), + definition_name=stored.ref.source_type, + definition_version=stored.value.definition_version, + ) + for stored in sources + ) + fingerprints = { + (stored.ref.source_type, stored.ref.source_id): stored.value.definition_fingerprint + for stored in sources + if isinstance(stored.value, SourceObservation) + } + return await self._bound_evidence(connection, scope_id, materials, fingerprints) + + async def historical_evidence( + self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / + ) -> tuple[SourceEvidence, ...]: + """Check retained material without loading today's native Definition. + + Only the stored representation selects decoding. Canonical JSON retains + all native fields, rather than round-tripping through a newer model. + This does not supply a decoder or provider value for ordinary reads. + """ + + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) + if not identities: + return () + rows = ( + ( + await connection.execute( + select(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == scope_id, + tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_(identities), + ) + ) + ) + .mappings() + .all() + ) + by_ref = {(row["source_type"], row["source_id"]): row for row in rows} + materials: list[SourceEvidence] = [] + fingerprints: dict[tuple[str, str], str] = {} + for ref in identities: + reference = SourceRef(source_type=ref[0], source_id=ref[1]) + row = by_ref.get(ref) + if row is None: + raise RepositoryNotFoundError("source", (scope_id, reference)) + source, material = await self._historical_material(connection, reference, row) + if isinstance(source, SourceObservation): + fingerprints[ref] = source.definition_fingerprint + elif ref[0] == CONTENT_SOURCE_NAME: + # The server-owned Content witness is the sole permitted material + # mutation; this exclusion is the same one used at acceptance. + internal = material.get("internal") + if ( + isinstance(internal, dict) + and internal.get("role") == "lineage_only" + and internal.get("operation") in {"artifact_create", "artifact_replace"} + ): + continue + material = {key: value for key, value in material.items() if key != "handoff_receipt"} + materials.append( + SourceEvidence( + source_ref=reference, + materialization=source.materialization, + material_digest=f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}", + definition_name=ref[0], + definition_version=source.definition_version, + ) + ) + return await self._bound_evidence(connection, scope_id, tuple(materials), fingerprints) + + async def _historical_material( + self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] + ) -> tuple[Source, dict[str, JsonValue]]: + """Validate the stored representation and its retained schema, not a live model.""" + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) + except InvalidStoredPayloadError: + # Bare legacy material has no representation discriminator. Use + # its registered decoder when available; otherwise retain opaque + # canonical fields for neutral maintenance, not a guessed schema. + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + return source, material + source = self._decode_row(row).value + return source, source.model_dump(mode="json", by_alias=True) + material = envelope.value + model = SourceObservation if envelope.representation == "observation" else Source + source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) + decoded = SourceRef( + source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, + source_id=source.name, + ) + if reference != decoded: + raise IdentityMismatchError("source", reference, decoded) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + fingerprint = await connection.scalar( + select(table.c.manifest_fingerprint).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) + ) + manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint == fingerprint: + try: + Draft202012Validator.check_schema(manifest.source_schema) + value = source.payload if isinstance(source, SourceObservation) else material + Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) + except JsonSchemaValidationError as error: + raise InvalidStoredPayloadError( + "source", reference.source_type, "payload does not match the historical schema" + ) from error + except (SchemaError, Unresolvable): + self._validate_unbound_material(reference, envelope, source) + else: + self._validate_unbound_material(reference, envelope, source) + return source, material + + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: + """Keep supported exact-read errors when no historical schema is available. + + A live decoder validates only material of its own version; it never + contributes a declaration or reinterprets a valid historical binding. + """ + + if envelope.representation == "observation": + return # SourceObservation already validated its envelope and payload. + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + return + if definition.version == source.definition_version: + self._decode_envelope(reference.source_type, envelope) + + @staticmethod + async def _bound_evidence( + connection: AsyncConnection, + scope_id: str, + materials: tuple[SourceEvidence, ...], + fingerprints: Mapping[tuple[str, str], str], + ) -> tuple[SourceEvidence, ...]: + if not materials: + return () table = SOURCE_EVIDENCE_BINDINGS_TABLE rows = ( ( @@ -274,7 +440,7 @@ async def evidence( select(table).where( table.c.scope_id == scope_id, tuple_(table.c.source_type, table.c.source_id).in_( - tuple((stored.ref.source_type, stored.ref.source_id) for stored in sources) + tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) ), ) ) @@ -285,33 +451,25 @@ async def evidence( bindings = {(row["source_type"], row["source_id"]): row for row in rows} manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} evidence: list[SourceEvidence] = [] - for stored in sources: - source = stored.value - digest = _source_material_digest(source) - value = SourceEvidence( - source_ref=stored.ref, - materialization=source.materialization, - material_digest=digest, - definition_name=stored.ref.source_type, - definition_version=source.definition_version, - ) - binding = bindings.get((stored.ref.source_type, stored.ref.source_id)) + for value in materials: + identity = (value.source_ref.source_type, value.source_ref.source_id) + binding = bindings.get(identity) if binding is None or ( - binding["material_digest"] != digest + binding["material_digest"] != value.material_digest or binding["definition_name"] != value.definition_name or binding["definition_version"] != value.definition_version or not isinstance(binding["manifest_fingerprint"], str) ): evidence.append(value) continue - identity = (value.definition_name, value.definition_version) - if identity not in manifests: - manifests[identity] = await _find_evidence_manifest(connection, *identity) - manifest = manifests[identity] + definition_identity = (value.definition_name, value.definition_version) + if definition_identity not in manifests: + manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) + manifest = manifests[definition_identity] if ( manifest is None or manifest.fingerprint != binding["manifest_fingerprint"] - or (isinstance(source, SourceObservation) and source.definition_fingerprint != manifest.fingerprint) + or (identity in fingerprints and fingerprints[identity] != manifest.fingerprint) ): evidence.append(value) continue diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index f2a2a14ed9..70688c79e7 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -58,7 +58,10 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository, atomic_memory_state_expression -from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -80,7 +83,6 @@ from powercontext.builtin.records import BaseOperationNotSupportedError, InvalidBaseAccessRequestError from powercontext.builtin.runtime.artifact_merge import ArtifactMergeAccess, merge_artifact_tags from powercontext.builtin.tags import TagFilter -from powercontext.sources.evidence import MemoryEvidenceDeclaration class AtomicMemoryAccess(ArtifactMergeAccess): @@ -237,45 +239,13 @@ async def evidence(self, artifact_id: str, *, revision: int, context=None) -> At application.database.transaction(consistent_snapshot=True) as connection, ): record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) - sources = application.artifacts.source_repository - materials = ( - () - if sources is None - else await sources.evidence(connection, self.scope_id, record.artifact.lineage.sources) - ) - annotation = await application.index.read_evidence(connection, self.scope_id, record.ref) - if annotation is not None: - by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} - for accepted in annotation.sources: - material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) - if material is not None and accepted.material_digest != material.material_digest: - raise AtomicMemoryIndexError( - "evidence-material", "Historical evidence does not match the exact Source material" - ) - if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( - item.source_ref for item in materials - ): - return AtomicMemoryEvidenceAnnotation( - scope_id=self.scope_id, - artifact_ref=record.ref, - sources=tuple( - material.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for material in materials - ), - ) - verified = tuple( - accepted - if ( - accepted.materialization == material.materialization - and accepted.definition_name == material.definition_name - and accepted.definition_version == material.definition_version - and accepted.manifest_fingerprint == material.manifest_fingerprint - and accepted.declaration == material.declaration - ) - else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for accepted, material in zip(annotation.sources, materials, strict=True) + return await AtomicMemoryEvidenceRepository().verify( + connection, + self.scope_id, + record.ref, + record.artifact.lineage.sources, + sources=application.artifacts.source_repository, ) - return annotation.model_copy(update={"sources": verified}) async def list( self, diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 90757129b0..67f429a9ca 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Explicit offline repair of only the current Atomic Memory projection.""" +"""Offline current-projection repair and verification of retained Atomic history.""" from __future__ import annotations @@ -25,12 +25,14 @@ from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryRecord, AtomicMemoryStateValue from powercontext.builtin.artifacts.memory.canonical import canonical_json from powercontext.builtin.artifacts.search import analyze_text from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -48,6 +50,7 @@ ) from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE @@ -114,6 +117,42 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) +async def _verify_history( + connection: AsyncConnection, artifacts: ArtifactRepository, batch_size: int, sources: SourceRepository | None +) -> dict[str, int]: + table = ARTIFACTS_TABLE + evidence = AtomicMemoryEvidenceRepository() + after: tuple[str, str, int] | None = None + revisions = neutral_sources = 0 + while True: + statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( + table.c.family == AtomicMemory.family + ) + if after is not None: + statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) + rows = ( + await connection.execute( + statement.order_by(table.c.scope_id, table.c.artifact_id, table.c.revision).limit(batch_size) + ) + ).all() + if not rows: + break + for row in rows: + scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) + reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) + artifact = await artifacts.get(connection, scope_id, reference) + annotation = await evidence.verify( + connection, scope_id, reference, artifact.lineage.sources, sources=sources + ) + revisions += 1 + neutral_sources += sum( + source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" + for source in annotation.sources + ) + after = scope_id, artifact_id, revision + return {"historical_revisions": revisions, "neutral_sources": neutral_sources} + + async def _record( connection: AsyncConnection, artifacts: ArtifactRepository, scope_id: str, artifact_id: str ) -> AtomicMemoryRecord: @@ -175,12 +214,20 @@ async def _projection_differences( async def verify_atomic_memory_current( - connection: AsyncConnection, index: AtomicMemoryIndex, /, *, batch_size: int = 100 + connection: AsyncConnection, + index: AtomicMemoryIndex, + /, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: - """Check every current identity against authority, including post-migration creations.""" + """Check current projections and every exact revision's retained evidence.""" + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") await _require_authority(connection) artifacts = ArtifactRepository((AtomicMemory,)) + history_counts = await _verify_history(connection, artifacts, batch_size, sources) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) errors: list[str] = [] @@ -218,7 +265,7 @@ async def verify_atomic_memory_current( return AtomicMemoryProjectionRebuildReport( ready=not errors, profile_fingerprint=fingerprint, - counts={"active_rows": active, "obsolete_rows": obsolete}, + counts={"active_rows": active, "obsolete_rows": obsolete, **history_counts}, errors=tuple(errors), ) @@ -241,6 +288,7 @@ async def rebuild_atomic_memory_projection( maintenance_confirmed: bool, embedding_model: EmbeddingModel | None = None, batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: """Repair current without changing revisions, Family state or retained history. @@ -309,7 +357,7 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete - report = await verify_atomic_memory_current(connection, index, batch_size=batch_size) + report = await verify_atomic_memory_current(connection, index, batch_size=batch_size, sources=sources) return report.model_copy( update={ "counts": { From 034e0cc5a8d661f928485fb243000763c612bb0c Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 16:34:18 +0800 Subject: [PATCH 03/26] fix(memory): enforce evidence schema safety and backend conformance --- .../persistence/atomic_memory_readiness.py | 10 +++- .../migrations/atomic_memory_evidence.py | 46 +++++++++++++++---- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index fdd46b5b74..455af19b99 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -26,6 +26,10 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( + AtomicMemoryEvidenceSchemaError, + require_atomic_memory_evidence_schema, +) LEGACY_CITATION_COLUMN = "memory_citations" _LEGACY_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") @@ -90,11 +94,15 @@ async def legacy_foreign_keys(connection: AsyncConnection, tables: set[str]) -> async def atomic_memory_readiness_issues( connection: AsyncConnection, tables: set[str] | None = None, *, collections_removed: bool = True ) -> list[str]: - """Check public migration completion, allowing collections only during offline pre-removal acceptance.""" + """Check versioned schema and public migration completion without reading retained history.""" if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py index cafc2cdfe9..72ef0935db 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -43,6 +43,7 @@ class AtomicMemoryEvidenceSchemaError(PersistenceError): def __init__(self, detail: str) -> None: message = { + "missing": "Atomic Memory evidence schema is incomplete; apply the registered p0004 migration", "version": "Atomic Memory evidence schema version is unsupported", "shape": "Atomic Memory evidence table has an incompatible shape", }[detail] @@ -63,16 +64,41 @@ async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" statements = json.loads(resources.read_text())[dialect] + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): - exists = await connection.run_sync(lambda sync, name=table.name: inspect(sync).has_table(name)) - if not exists: - await connection.exec_driver_sql(statement) - await _require_shape(connection, table.name, statement, dialect) + if table.name in tables: + await _require_shape(connection, table.name, statement, dialect) marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE - versions = tuple((await connection.execute(select(marker.c.schema_version))).scalars()) + versions = ( + tuple((await connection.execute(select(marker.c.schema_version))).scalars()) if marker.name in tables else () + ) + if versions and versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + if versions and any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + # A ready marker cannot authorize recreating missing retained history. + raise AtomicMemoryEvidenceSchemaError("missing") + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name not in tables: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) if not versions: await connection.exec_driver_sql(statements[-1]) - elif versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + + +async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tables: set[str] | None = None) -> None: + """Check retained evidence readiness without DDL or historical payload reads.""" + + if tables is None: + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + if any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + raise AtomicMemoryEvidenceSchemaError("missing") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + await _require_shape(connection, table.name, statement, dialect) + versions = tuple((await connection.execute(select(ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE.c.schema_version))).scalars()) + if versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): raise AtomicMemoryEvidenceSchemaError("version") @@ -88,11 +114,15 @@ async def _require_shape(connection: AsyncConnection, name: str, statement: str, return # Reuse the frozen MySQL inventory parser so physical formatting accepted # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] - actual_name, actual_table = _parse_table(actual) - expected_name, expected_table = _parse_table(statement) + try: + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + except MigrationError as error: + raise AtomicMemoryEvidenceSchemaError("shape") from error if not _matches( {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, From 4d1bf17abba42774d189d28eb46d768100ffa497 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 16:38:37 +0800 Subject: [PATCH 04/26] fix(memory): preserve unrelated database inspection --- .../builtin/persistence/atomic_memory_readiness.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index 455af19b99..52df67b4a5 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -25,6 +25,7 @@ from sqlalchemy import inspect, text from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.errors import PersistenceError from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( AtomicMemoryEvidenceSchemaError, @@ -99,10 +100,11 @@ async def atomic_memory_readiness_issues( if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] - try: - await require_atomic_memory_evidence_schema(connection, tables) - except AtomicMemoryEvidenceSchemaError as error: - issues.append(str(error)) + if "pc_artifacts" in tables or any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables From cf8068a0f883e73ca78f2b7e4d5d794612b74584 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 01:02:50 +0800 Subject: [PATCH 05/26] test(memory): consolidate source evidence E2E acceptance Cover persisted declarations, scope isolation, historical neutrality, synthetic model trust claims, and unchanged recall consumers on SQLite and OceanBase. --- tests/e2e/conftest.py | 13 + ...t_atomic_memory_evidence_neutral_recall.py | 131 ++++++++++ .../test_atomic_memory_projection_rebuild.py | 17 +- .../e2e/test_atomic_memory_source_evidence.py | 228 ++++++++++++++++++ tests/e2e/test_observability.py | 9 +- 5 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/test_atomic_memory_evidence_neutral_recall.py create mode 100644 tests/e2e/test_atomic_memory_source_evidence.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index bdb820e071..a1f2778fa8 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import json import os from collections.abc import Iterator from pathlib import Path @@ -26,6 +27,7 @@ from pydantic import SecretStr from sqlalchemy.engine import make_url +from powercontext.builtin.persistence.migrations.backup import identify_fork_server from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -63,6 +65,17 @@ async def manage(*, create: bool) -> None: nonlocal owned async with migration_profile(config) as profile, profile.database.transaction() as connection: assert (await connection.exec_driver_sql("SELECT DATABASE()")).scalar_one() == url.database + version = str((await connection.exec_driver_sql("SELECT VERSION()")).scalar_one()) + comment = str((await connection.exec_driver_sql("SELECT @@version_comment")).scalar_one()) + product, engine_version = identify_fork_server(version, comment) + if product != "oceanbase_ai" or engine_version is None: + pytest.fail( + "Acceptance requires an identified real OceanBase version, not MySQL or SeekDB", pytrace=False + ) + if create: + evidence = {"backend": "oceanbase", "version": version, "engine_version": engine_version} + request.node.user_properties.append(("backend_identity", evidence)) + print(json.dumps(evidence, ensure_ascii=False)) await connection.exec_driver_sql( f"CREATE DATABASE `{database}`" if create else f"DROP DATABASE `{database}`" ) diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py new file mode 100644 index 0000000000..c3ad6ee50b --- /dev/null +++ b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accepted Source declarations do not alter recall or consumer revision citations.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import httpx +import pytest + +from powercontext.builtin.artifacts.handoff import PrepareHandoff +from powercontext.builtin.runtime import ( + BuiltinConfig, + HandoffArtifactCitation, + PrepareContextRequest, + RuntimeConfig, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.server.app import ServerApplication, create_app +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.test_atomic_memory_source_evidence import _commit +from tests.e2e.test_handoff_runtime import _EchoHandoffPipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +def test_source_declarations_preserve_recall_and_consumers(migration_backend: MigrationBackend) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=(), atomic_memory_related_mode="fts"), + ) + arms = [] + neutral = MemoryEvidenceDeclaration() + candidates = ( + (SourceRef(source_type="note", source_id="note"), "memory-a", "alpha acceptance"), + (SourceRef(source_type="content", source_id="source"), "memory-b", "alpha guidance"), + ) + for declaration in ( + neutral, + MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ): + registry = SourceDefinitionRegistry(( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration), + AdapterSourceDefinition(NoteAdapter()), + )) + async with open_builtin_runtime( + config, + scheduler_path=migration_backend.directory / "consumer-scheduler.db", + source_registry=registry, + handoff_pipeline=_EchoHandoffPipeline(), + ) as runtime: + assert runtime.scopes is not None and runtime.atomic_memory is not None + scope = await runtime.scopes.create( + ScopeDraft( + title="Recall", + summary="Evidence neutrality.", + idempotency_key=declaration.authority, + ) + ) + scope_id = scope.scope_id + async with open_builtin_contexts(config, source_registry=registry) as contexts: + await contexts.records.capture_source(scope_id, "content", "source", "alpha guidance", {}) + context = await contexts.get(scope_id) + note = await context.sources.resolve(NoteInput(note_id="note", body="alpha acceptance")) + await context.sources.add(note) + memories = [] + for source, artifact_id, text in candidates: + memory = await _commit(contexts, scope_id, artifact_id, text, sources=(source,)) + annotation = await contexts.atomic_memory.for_scope(scope_id).evidence(artifact_id, revision=1) + assert annotation.artifact_ref == memory.ref + expected = declaration if source.source_type == "content" else neutral + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (source, expected), + ) + memories.append(memory) + page = await runtime.atomic_memory.for_scope(scope_id).search("alpha", mode="text", limit=2) + assert tuple((hit.hit.artifact_ref, hit.text, hit.matched_by) for hit in page.hits) == tuple( + (memory.ref, text, ("text",)) for memory, (_, _, text) in zip(memories, candidates, strict=True) + ) + recalled = tuple((hit.hit.artifact_ref.model_dump(mode="json"), hit.text) for hit in page.hits) + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + response = await client.post("/v1/memory/search", json={"scope_id": scope_id, "query": "alpha"}) + assert response.status_code == 200, response.text + public = response.json() + assert ( + tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled + ) + prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) + assert prepared.status == "ready" and prepared.content is not None + items = json.loads(prepared.content.splitlines()[-2])["items"] + assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled + citation = HandoffArtifactCitation(artifact_ref=page.hits[-1].hit.artifact_ref) + handoffs = runtime.handoff.for_scope(scope_id) + finalized = await handoffs.finalize( + await handoffs.prepare( + PrepareHandoff(objective="Keep the recalled revision.", evidence=(citation,)) + ) + ) + assert finalized.content.state[0].citations == (citation,) + assert finalized.content.next_action is not None + assert finalized.content.next_action.citations == (citation,) + arms.append((page.hits, public, prepared.content.replace(scope_id, "scope"), finalized.content)) + assert arms[0] == arms[1] + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 3075699659..5263ff1a0c 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +from dataclasses import replace from pathlib import Path import pytest @@ -29,6 +30,9 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import SourceDefinitionRegistry +from powercontext.sources.evidence import MemoryEvidenceDeclaration from tests.e2e.atomic_memory_migration_fixture import SCOPE, _atomic, _seed_and_migrate from tests.legacy_memory import add_legacy_citation_columns @@ -36,6 +40,12 @@ @pytest.mark.parametrize("retention", ["complete", "archive_only", "entries_only", "removed", "unreadable"]) def test_rebuild_uses_current_authority_independently_of_legacy_retention(tmp_path: Path, retention: str) -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'projection.db'}") + strong_registry = SourceDefinitionRegistry(( + replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ), + )) async def scenario() -> None: await _seed_and_migrate(config) @@ -43,6 +53,8 @@ async def scenario() -> None: memory = contexts.atomic_memory.for_scope(SCOPE) current = await memory.get(_atomic("alpha", 2).artifact_id) historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) + annotation = await memory.evidence(historical.ref.artifact_id, revision=1) + assert all(source.declaration == MemoryEvidenceDeclaration() for source in annotation.sources) async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: if retention in {"entries_only", "removed"}: @@ -63,10 +75,13 @@ async def scenario() -> None: ) assert report.ready, report.errors assert report.counts["rebuilt_rows"] == 2 - async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + assert report.counts["historical_revisions"] == 3 + assert report.counts["neutral_sources"] == 4 + async with open_builtin_contexts(BuiltinConfig(database=config), source_registry=strong_registry) as contexts: memory = contexts.atomic_memory.for_scope(SCOPE) assert await memory.get(current.ref.artifact_id) == current assert await memory.get(historical.ref.artifact_id, revision=1) == historical + assert await memory.evidence(historical.ref.artifact_id, revision=1) == annotation result = await memory.search("Tasks A and C", mode="text") assert current.ref in tuple(hit.hit.artifact_ref for hit in result.hits) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py new file mode 100644 index 0000000000..5fe44a12e3 --- /dev/null +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -0,0 +1,228 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned source evidence acceptance on disposable SQLite and OceanBase.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace + +import pytest +from sqlalchemy import event, text +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import ArtifactLineage +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.extraction import ( + AtomicMemoryCandidate, + AtomicMemoryExtractionOutput, +) +from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.runtime import BuiltinConfig, SubmitSourceObservation, open_builtin_contexts +from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from powercontext.sources.observations import manifest_for_definition, project_source_for_transport +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.dream_support import atomic_memory_pipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), artifacts=(), expected_revision=None): + application = contexts.atomic_memory + async with contexts.database.transaction() as connection: + plan = await application.service.inspect_change( + connection, + scope_id, + artifact_id, + AtomicMemoryContent(kind="fact", text=text_value), + None, + expected_revision=expected_revision, + lineage=ArtifactLineage(sources=sources, artifacts=artifacts), + ) + prepared = await application.service.prepare_change(plan) + async with contexts.database.transaction() as connection: + result = await application.service.commit(connection, prepared, None) + return result.primary + + +def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: + strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") + remote = MemoryEvidenceDeclaration(authority="system_attested", verification="verified") + + class RemoteNoteAdapter(NoteAdapter): + name = "remote-note" + + worker = SourceDefinitionRegistry((AdapterSourceDefinition(RemoteNoteAdapter(), memory_evidence=remote),)) + manifest = manifest_for_definition(worker.definition_for_name("remote-note")) + registry = SourceDefinitionRegistry( + ( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=strong), + AdapterSourceDefinition(NoteAdapter(), memory_evidence=asserted), + ), + remote_declarations={(manifest.name, manifest.version, manifest.fingerprint): remote}, + ) + config = BuiltinConfig(database=migration_backend.config) + refs = ( + SourceRef(source_type="content", source_id="cafe\u0301"), + SourceRef(source_type="note", source_id="native"), + SourceRef(source_type="remote-note", source_id="remote"), + ) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + await contexts.records.capture_source( + "scope", "content", "cafe\u0301", "Release received.", {"authority": "untrusted"} + ) + native = await context.sources.resolve(NoteInput(note_id="native", body="User correction.")) + await context.sources.add(native) + source = await worker.resolve(NoteInput(note_id="remote", body="Remote release evidence.")) + await contexts.register_source_definition(manifest) + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope", observation=project_source_for_transport(worker, source)) + ) + original = await _commit(contexts, "scope", "rule", "Release rule one.", sources=refs) + memory = contexts.atomic_memory.for_scope("scope") + annotation = await memory.evidence("rule", revision=1) + assert annotation.artifact_ref == original.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (refs[0], strong), + (refs[1], asserted), + (refs[2], remote), + ) + + def reject_snapshot(connection, cursor, statement, *args): + if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): + raise SQLAlchemyError(statement) + + engine = contexts.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", reject_snapshot) + try: + with pytest.raises(SQLAlchemyError): + await _commit(contexts, "scope", "rule", "Failed revision.", expected_revision=1) + finally: + event.remove(engine, "before_cursor_execute", reject_snapshot) + assert await memory.get("rule") == original + revised = await _commit(contexts, "scope", "rule", "Rule two.", sources=(refs[2],), expected_revision=1) + current = await memory.evidence("rule", revision=2) + assert current.sources == (annotation.sources[2],) + neutral_registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION, AdapterSourceDefinition(NoteAdapter()))) + async with open_builtin_contexts(config, source_registry=neutral_registry) as contexts: + await contexts.records.capture_source( + "other", refs[0].source_type, refs[0].source_id, "Other Scope release.", {} + ) + other = await _commit(contexts, "other", "rule", "Other Scope rule.", sources=(refs[0],)) + other_annotation = await contexts.atomic_memory.for_scope("other").evidence("rule", revision=1) + assert other_annotation.scope_id == "other" + assert tuple((item.source_ref, item.declaration) for item in other_annotation.sources) == ( + (refs[0], MemoryEvidenceDeclaration()), + ) + assert other_annotation.sources[0].material_digest != annotation.sources[0].material_digest + + async def assert_retained(contexts, historical): + memory = contexts.atomic_memory.for_scope("scope") + assert await memory.evidence("rule", revision=1) == historical + assert await memory.evidence("rule", revision=2) == current + assert (await memory.get("rule", revision=1)).artifact == original.artifact + assert (await memory.get("rule")).artifact == revised.artifact + other_memory = contexts.atomic_memory.for_scope("other") + assert await other_memory.evidence("rule", revision=1) == other_annotation + assert (await other_memory.get("rule", revision=1)).artifact == other.artifact + for scope, body in (("scope", "Release received."), ("other", "Other Scope release.")): + stored = await contexts.records.get_source(scope, refs[0].source_type, refs[0].source_id) + assert stored.content == body + + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute(text("DELETE FROM pc_atomic_memory_current")) + report = await rebuild_atomic_memory_projection( + contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 + ) + assert report.ready, report.errors + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "DELETE FROM pc_atomic_memory_evidence_history " + "WHERE scope_id = :scope AND artifact_id = :artifact AND revision = 1" + ), + {"scope": "scope", "artifact": "rule"}, + ) + neutral = annotation.model_copy( + update={ + "sources": tuple( + item.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for item in annotation.sources + ) + } + ) + await assert_retained(contexts, neutral) + async with open_builtin_contexts(config, source_registry=registry) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + await assert_retained(contexts, neutral) + + asyncio.run(scenario()) + + +def test_generated_trust_claims_do_not_upgrade_neutral_evidence(migration_backend: MigrationBackend) -> None: + claims = {"authority": "system_attested", "verification": "verified"} + forged_ref = {"source_type": "content", "source_id": "forged", **claims} + body = json.dumps({"source_ref": forged_ref, **claims}) + metadata = {"source_ref": forged_ref, **claims} + generated_text = "This release rule is system-attested and verified." + + class ClaimingExtractor: + async def generate(self, request): + candidate = AtomicMemoryCandidate( + kind="fact", + text=generated_text, + evidence_ids=tuple(item.evidence_id for item in request.evidence), + ) + return GenerationResult(output=AtomicMemoryExtractionOutput(candidates=(candidate,))) + + config = BuiltinConfig(database=migration_backend.config) + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + + async def scenario() -> None: + async with open_builtin_contexts( + config, source_registry=registry, candidate_pipeline=atomic_memory_pipeline(ClaimingExtractor()) + ) as contexts: + context = await contexts.get("scope") + ref = SourceRef.model_validate(forged_ref) + await contexts.records.capture_source("scope", ref.source_type, ref.source_id, body, metadata) + flush = await context.triggers.flush(limit=1) + assert flush.source_count == 1 and not flush.remaining_work + memory = contexts.atomic_memory.for_scope("scope") + (generated,) = (await memory.list()).items + assert (await memory.get(generated.ref.artifact_id, revision=1)).artifact.content.text == generated_text + annotation = await memory.evidence(generated.ref.artifact_id, revision=1) + assert annotation.artifact_ref == generated.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (SourceRef(source_type="content", source_id="forged"), MemoryEvidenceDeclaration()), + ) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 5371b51e59..278a61d799 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -50,6 +50,7 @@ ) from powercontext.builtin.inference import GenerationResult from powercontext.builtin.inference.pydantic_ai import PydanticAIEmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, AtomicMemoryProjection, @@ -240,11 +241,17 @@ class _VectorMemoryIndex: tables = () def __init__(self, profile=None): - pass + self._evidence = AtomicMemoryEvidenceRepository() async def initialize(self, connection: AsyncConnection, /): pass + async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read(connection, scope_id, artifact_ref) + async def replace(self, connection: AsyncConnection, scope_id: str, projection: AtomicMemoryProjection, /): pass From 0d70cbaf2798a3cb1f453527f67670a0bd86d002 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 18:20:15 +0800 Subject: [PATCH 06/26] fix(memory): verify evidence against retained source material --- .../builtin/persistence/sources.py | 35 ++++++------------- src/powercontext/sources/observations.py | 2 +- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 597c89a842..0b58316aa7 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -266,28 +266,7 @@ async def evidence( value is substituted for historical registration facts. """ - stored_sources = await self.get_many(connection, scope_id, refs) - # These server-owned records describe a management write, not evidence - # supplied for that write. Other lineage-only Sources remain represented. - sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) - if not sources: - return () - materials = tuple( - SourceEvidence( - source_ref=stored.ref, - materialization=stored.value.materialization, - material_digest=_source_material_digest(stored.value), - definition_name=stored.ref.source_type, - definition_version=stored.value.definition_version, - ) - for stored in sources - ) - fingerprints = { - (stored.ref.source_type, stored.ref.source_id): stored.value.definition_fingerprint - for stored in sources - if isinstance(stored.value, SourceObservation) - } - return await self._bound_evidence(connection, scope_id, materials, fingerprints) + return await self.historical_evidence(connection, scope_id, refs) async def historical_evidence( self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / @@ -341,7 +320,7 @@ async def historical_evidence( SourceEvidence( source_ref=reference, materialization=source.materialization, - material_digest=f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}", + material_digest=_source_material_digest_material(material), definition_name=ref[0], definition_version=source.definition_version, ) @@ -372,7 +351,11 @@ async def _historical_material( raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error return source, material source = self._decode_row(row).value - return source, source.model_dump(mode="json", by_alias=True) + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + return source, material material = envelope.value model = SourceObservation if envelope.representation == "observation" else Source source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) @@ -756,6 +739,10 @@ async def _find_evidence_manifest( return None +def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + def _dump_source( source: Source, *, diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 576e29fb79..a7b1858c03 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -163,7 +163,7 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: """Build the immutable declaration transported by a remote worker.""" - source_schema = _json_object(definition.source_class.model_json_schema()) + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), From 3305b6bb1edb9b1c0b86b3ec4955239dad9d36d8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 21:17:41 +0800 Subject: [PATCH 07/26] fix(sources): align remote payload aliases with manifests --- src/powercontext/sources/observations.py | 2 +- tests/builtin/persistence/test_sources.py | 53 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index a7b1858c03..c9bb8a802f 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -199,7 +199,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json")) + payload = _json_object(source.model_dump(mode="json", by_alias=True)) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 22e080499f..1e88d9bb96 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -17,6 +17,7 @@ import asyncio import pytest +from pydantic import BaseModel, Field from sqlalchemy import insert from powercontext import InvalidSourceDefinitionError @@ -29,8 +30,11 @@ from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE +from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( + AdapterSourceDefinition, + Source, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -46,6 +50,31 @@ ) +class SerializationAliasInput(BaseModel): + source_id: str + body: str + + +class SerializationAliasSource(Source): + body: str = Field(serialization_alias="wire_text") + + +class SerializationAliasAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = SerializationAliasSource + + async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: + return SerializationAliasSource( + name=value.source_id, + materialization=SourceMaterialization.CAPTURED, + body=value.body, + ) + + async def read(self, source: SerializationAliasSource, /) -> str: + return source.body + + def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -149,6 +178,30 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_observation_ingestion_accepts_serialization_aliases() -> None: + async def scenario() -> None: + worker = SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter()),)) + source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) + manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) + observation = project_source_for_transport(worker, source) + + async with repository_profile() as (profile, _repositories): + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + context = await contexts.get("scope-a") + stored = await context.sources.get(observation) + + assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") + assert isinstance(stored, SourceObservation) + assert stored.payload["wire_text"] == "Remote note text." + assert "body" not in stored.payload + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): From c9ee82d5dc9cf75f2e7c8800959a611c507a7d03 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Fri, 16 Oct 2026 17:16:32 +0800 Subject: [PATCH 08/26] fix(sources): preserve envelope keys with alias generators --- src/powercontext/sources/models.py | 11 ++++--- tests/builtin/persistence/test_sources.py | 37 ++++++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6e2cd48df6..6235d78f9e 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -16,7 +16,7 @@ from enum import StrEnum -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from powercontext.errors import InvalidSourceReferenceError from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH @@ -60,10 +60,11 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - name: str - definition_version: str = "1" - materialization: SourceMaterialization - description: str | None = None + # Envelope keys remain canonical when a Definition generates aliases for its own fields. + name: str = Field(serialization_alias="name") + definition_version: str = Field(default="1", serialization_alias="definition_version") + materialization: SourceMaterialization = Field(serialization_alias="materialization") + description: str | None = Field(default=None, serialization_alias="description") @field_validator("definition_version") @classmethod diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 1e88d9bb96..b74a798a7e 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -17,7 +17,8 @@ import asyncio import pytest -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel from sqlalchemy import insert from powercontext import InvalidSourceDefinitionError @@ -59,15 +60,27 @@ class SerializationAliasSource(Source): body: str = Field(serialization_alias="wire_text") +class CamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class UpperCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) + + class SerializationAliasAdapter: name = "serialization-alias" input_class = SerializationAliasInput - source_class = SerializationAliasSource + + def __init__(self, source_class: type[SerializationAliasSource]) -> None: + self.source_class = source_class async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: - return SerializationAliasSource( + return self.source_class( name=value.source_id, + definition_version="2", materialization=SourceMaterialization.CAPTURED, + description="Remote alias description.", body=value.body, ) @@ -178,9 +191,14 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_remote_observation_ingestion_accepts_serialization_aliases() -> None: +@pytest.mark.parametrize("source_class", (SerializationAliasSource, CamelCaseSource, UpperCaseSource)) +def test_remote_observation_ingestion_accepts_serialization_aliases( + source_class: type[SerializationAliasSource], +) -> None: async def scenario() -> None: - worker = SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter()),)) + worker = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(source_class), version="2"), + )) source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) observation = project_source_for_transport(worker, source) @@ -196,8 +214,13 @@ async def scenario() -> None: assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") assert isinstance(stored, SourceObservation) - assert stored.payload["wire_text"] == "Remote note text." - assert "body" not in stored.payload + assert stored.payload == { + "name": "remote-1", + "definition_version": "2", + "materialization": "captured", + "description": "Remote alias description.", + "wire_text": "Remote note text.", + } asyncio.run(scenario()) From dc9b6a175154be0a2c334e7c02c99c2df0da3651 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Sat, 17 Oct 2026 13:28:08 +0800 Subject: [PATCH 09/26] fix(sources): align envelope validation with serialization --- src/powercontext/sources/models.py | 10 +++--- tests/builtin/persistence/test_sources.py | 40 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6235d78f9e..0c31b8a387 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -60,11 +60,11 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - # Envelope keys remain canonical when a Definition generates aliases for its own fields. - name: str = Field(serialization_alias="name") - definition_version: str = Field(default="1", serialization_alias="definition_version") - materialization: SourceMaterialization = Field(serialization_alias="materialization") - description: str | None = Field(default=None, serialization_alias="description") + # Envelope keys stay canonical for validation and serialization, independently of adapter aliases. + name: str = Field(alias="name") + definition_version: str = Field(default="1", alias="definition_version") + materialization: SourceMaterialization = Field(alias="materialization") + description: str | None = Field(default=None, alias="description") @field_validator("definition_version") @classmethod diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index b74a798a7e..0f66d408f5 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -68,6 +68,12 @@ class UpperCaseSource(SerializationAliasSource): model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) +class NativeCamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel) + + body: str + + class SerializationAliasAdapter: name = "serialization-alias" input_class = SerializationAliasInput @@ -225,6 +231,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_native_generated_alias_source_survives_read_replay_and_restart(tmp_path) -> None: + async def scenario() -> None: + adapter = SerializationAliasAdapter(NativeCamelCaseSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + value = SerializationAliasInput(source_id="native-2", body="Keep the native version and body.") + source = await adapter.resolve(value) + # Keep the persisted version nondefault regardless of the constructor's alias rules. + source.definition_version = "2" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'native-alias.db'}") + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + assert await context.sources.add(source) == source + loaded = await context.sources.get(source) + assert loaded == source + assert await context.sources.read(loaded) == value.body + assert await context.sources.add(source) == source + assert await context.sources.journal.position(source) == 1 + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + loaded = await context.sources.get(source) + assert loaded == source + assert loaded.definition_version == "2" + resolved = await context.sources.resolve(value) + assert resolved == source + assert await context.sources.add(resolved) == source + assert await context.sources.journal.position(resolved) == 1 + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): From 8ee3aba7e99f34c1f296678a3fd3e3ff2f602b45 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Sun, 18 Oct 2026 15:17:15 +0800 Subject: [PATCH 10/26] fix(memory): preserve source identity and shared evidence Retain native Source identity independently of adapter aliases and mutable description defaults, preserve legacy read/replay behavior, and keep serialized business fields intact. Capture evidence at the shared Atomic Memory body-commit boundary, including restoration revisions, without retroactively attesting state-only writes. Refs #1975. --- .../artifacts/atomic_memory/service.py | 35 ++++- .../builtin/persistence/sources.py | 144 +++++++++++++++++- tests/builtin/persistence/test_sources.py | 90 +++++++++++ .../e2e/test_atomic_memory_source_evidence.py | 52 ++++++- 4 files changed, 308 insertions(+), 13 deletions(-) diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 90fb984ef4..94c049493f 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Protocol, cast from sqlalchemy.ext.asyncio import AsyncConnection +from typing_extensions import override from powercontext.artifacts import ArtifactLineage, ArtifactRef from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -52,6 +53,7 @@ from powercontext.builtin.artifacts.merge import ArtifactMergeSecurity, ArtifactMergeService, MergeTags from powercontext.builtin.artifacts.merge_models import ( ArtifactMergeErrors, + ArtifactMergeMutationResult, ArtifactMergePlan, ArtifactMergeRead, ArtifactMergeRecord, @@ -278,6 +280,32 @@ async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: await self.projections.remove(connection, scope_id, artifact_id) +class _AtomicMemoryMergeService(ArtifactMergeService): + """Accept evidence with every new body, including retired restoration results. + + Shared merges, the Atomic facade and processing all use this commit seam. + State-only writes must not attest old bodies. Capture joins the caller's + transaction so its failure rolls back authority, governance and projections. + """ + + @override + async def _commit_locked( + self, + connection: AsyncConnection, + prepared: PreparedArtifactMerge, + current: dict[str, ArtifactMergeRecord], + context: Any, + direct_source: SourceRef | None, + ) -> ArtifactMergeMutationResult: + result = await super()._commit_locked(connection, prepared, current, context, direct_source) + revisions = {write.artifact_id for write in prepared.plan.writes if write.draft is not None} + projections = cast(AtomicMemoryMergeAdapter, self.adapter).projections + for record in result.records: + if record.ref.artifact_id in revisions: + await projections.capture_evidence(connection, prepared.plan.scope_id, _atomic_record(record)) + return result + + class _AtomicPreviewAdapter: def __init__(self, signer): self.signer = signer @@ -304,7 +332,7 @@ def __init__( ) -> None: self.artifacts, self.states, self.security = artifacts, states, security self.projections, self.merge_tags, self.preview_signer = projections, merge_tags, preview_signer - self.shared = ArtifactMergeService( + self.shared = _AtomicMemoryMergeService( artifacts=artifacts, adapter=AtomicMemoryMergeAdapter(projections), security=security, @@ -481,11 +509,6 @@ async def commit_window( read_set=tuple(_common_read(read) for read in read_set), direct_source=direct_source, ) - for item, result in zip(prepared, results, strict=True): - revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} - for record in result.records: - if record.artifact.artifact_id in revisions: - await self.projections.capture_evidence(connection, scope_id, _atomic_record(record)) return tuple( AtomicMemoryMutationResult( changed=result.changed, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 0b58316aa7..24cff31caf 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -25,7 +25,7 @@ from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from pydantic import AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter from referencing import Registry from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update @@ -79,6 +79,9 @@ class _StoredSourcePayload(BaseModel): encoding: Literal["powercontext-source-v1"] representation: Literal["native", "observation"] value: dict[str, JsonValue] + # Server-owned header permits historical reads without adapter code. Never hash it. + source: Source | None = None + source_keys: dict[str, str] | None = None class StoredSource(BaseModel): @@ -158,6 +161,8 @@ async def add_with_status( return StoredSource(ref=ref, value=source, journal_position=stored.journal_position), False if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) + if representation == "native": + await self._retain_native_header(connection, scope_id, ref, existing, source) return stored, False position = await _next_journal_position(connection, scope_id) @@ -357,8 +362,7 @@ async def _historical_material( raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error return source, material material = envelope.value - model = SourceObservation if envelope.representation == "observation" else Source - source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) + source = self._historical_source(reference, envelope) decoded = SourceRef( source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, source_id=source.name, @@ -391,6 +395,40 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) return source, material + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: + """Decode identity independently of retained material and today's business defaults.""" + + material = envelope.value + if envelope.representation == "observation": + return load_model(SourceObservation, rfc8785.dumps(material), kind="source", name=reference.source_type) + if envelope.source is not None: + keys = envelope.source_keys or {key: key for key in ("name", "definition_version", "materialization")} + for key, alias in keys.items(): + value = getattr(envelope.source, key) + if alias not in material or material[alias] != value: + raise IdentityMismatchError("source-envelope", value, material.get(alias)) + return envelope.source + # Legacy aliased rows need their registered decoder once; canonical rows do not. + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + if {"name", "definition_version", "materialization"} <= material.keys(): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + header = material + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + header = _normalize_native_material(definition.source_class, material) + try: + return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) + except InvalidStoredPayloadError: + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: """Keep supported exact-read errors when no historical schema is available. @@ -665,6 +703,41 @@ def _decode_row(self, row: Mapping[Any, Any]) -> StoredSource: journal_position=int(row["journal_position"]), ) + async def _retain_native_header( + self, connection: AsyncConnection, scope_id: str, ref: SourceRef, row: Mapping[Any, Any], source: Source + ) -> None: + """Enrich only an identical replay's unreconstructible legacy header. + + The retained value, binding, manifest, digest and journal position never change. + This grants no authority and does not publish another processing intent. + """ + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=ref.source_type) + except InvalidStoredPayloadError: + envelope = _StoredSourcePayload( + encoding="powercontext-source-v1", + representation="native", + value=_JSON_OBJECT.validate_json(payload, strict=True), + ) + if envelope.source is not None: + return + header = _source_header(source) + with suppress(InvalidStoredPayloadError): + if load_model(Source, rfc8785.dumps(envelope.value), kind="source", name=ref.source_type) == header: + return + enriched = envelope.model_copy(update={"source": header, "source_keys": _source_keys(source, envelope.value)}) + await connection.execute( + update(SOURCES_TABLE) + .where( + SOURCES_TABLE.c.scope_id == scope_id, + SOURCES_TABLE.c.source_type == ref.source_type, + SOURCES_TABLE.c.source_id == ref.source_id, + ) + .values(payload=rfc8785.dumps(enriched.model_dump(mode="json", exclude_none=True))) + ) + def _decode_envelope( self, source_type: str, @@ -683,11 +756,15 @@ def _decode_envelope( definition = self._definition_by_name(source_type) source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, envelope.value)), kind="source", name=source_type, ) self._registry.definition_for_source(source) + if envelope.source is not None: + for key in ("name", "definition_version", "materialization"): + if getattr(envelope.source, key) != getattr(source, key): + raise IdentityMismatchError("source-envelope", envelope.source, _source_header(source)) return source, SourceRef(source_type=definition.name, source_id=source.name) def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Source, SourceRef]: @@ -702,9 +779,13 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour ) decoded = SourceRef(source_type=source.source_type, source_id=source.name) else: + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", source_type, "payload is not JSON") from error source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, material)), kind="source", name=source_type, ) @@ -743,6 +824,55 @@ def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" +def _source_header(source: Source) -> Source: + return Source(**{key: getattr(source, key) for key in Source.model_fields}) + + +def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: + """Retain declared identity spellings, including pre-canonical generated keys.""" + keys = { + key: type(source).model_fields[key].serialization_alias or key + for key in ("name", "definition_version", "materialization") + } + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if material is not None and generator is not None: + for key, alias in keys.items(): + if alias not in material: + keys[key] = generator(key) + return keys + + +def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: + """Read pre-canonical inherited envelope aliases, never business-field aliases. + + Derive spellings from the registered generator, not a version/key allowlist. + Explicit adapter aliases and conflicting identities retain strict validation. + """ + + generator = source_class.model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is None: + return material + normalized = dict(material) + for key in Source.model_fields: + if source_class.model_fields[key].validation_alias != key: + continue + alias = generator(key) + if any( + field_name != key and alias in (field.alias, field.validation_alias, field.serialization_alias) + for field_name, field in source_class.model_fields.items() + ): + continue + if alias != key and alias in normalized: + if key in normalized and normalized[key] != normalized[alias]: + raise IdentityMismatchError("source", normalized[key], normalized[alias]) + normalized[key] = normalized.pop(alias) + return normalized + + def _dump_source( source: Source, *, @@ -754,8 +884,10 @@ def _dump_source( encoding="powercontext-source-v1", representation=representation, value=_JSON_OBJECT.validate_python(source.model_dump(mode="json", by_alias=True)), + source=None if representation == "observation" else _source_header(source), + source_keys=None if representation == "observation" else _source_keys(source), ) - return rfc8785.dumps(envelope.model_dump(mode="json")) + return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 0f66d408f5..cb1e0a5521 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import json import pytest from pydantic import BaseModel, ConfigDict, Field @@ -35,6 +36,7 @@ from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceDefinitionRegistry, SourceMaterialization, @@ -265,6 +267,94 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(NativeCamelCaseSource), version="2"), + )) + repository = SourceRepository(registry) + source = NativeCamelCaseSource( + name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + material["definitionVersion"] = material.pop("definition_version") + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with ( + repository_profile() as (profile, _repositories), + profile.database.transaction() as connection, + ): + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=source.name, + payload=payload, + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + loaded = await repository.get(connection, "scope-a", ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == loaded + assert await repository.journal_position(connection, "scope-a") == 1 + + asyncio.run(scenario()) + + +def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: + class AliasedSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + business_name: str = Field(alias="name") + description: str | None = Field(default="captured description", exclude=True) + + class UpdatedSource(AliasedSource): + description: str | None = Field(default="current description", exclude=True) + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition( + SerializationAliasAdapter(AliasedSource), + version="2", + memory_evidence=declaration, + ), + )) + source = AliasedSource( + source_name="native-alias", + name="business label", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'explicit-alias.db'}") + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add(connection, "scope-a", source) + assert (await repository.get(connection, "scope-a", stored.ref)).value == source + assert await repository.add(connection, "scope-a", source) == stored + evidence = await repository.evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].declaration == declaration + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + updated = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(UpdatedSource), version="2"), + )) + loaded = await SourceRepository(updated).get(connection, "scope-a", stored.ref) + assert loaded.value.description == "current description" + assert await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) == evidence + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 5fe44a12e3..7e0178c5df 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -19,6 +19,7 @@ import asyncio import json from dataclasses import replace +from typing import cast import pytest from sqlalchemy import event, text @@ -31,8 +32,15 @@ AtomicMemoryExtractionOutput, ) from powercontext.builtin.inference import GenerationResult -from powercontext.builtin.runtime import BuiltinConfig, SubmitSourceObservation, open_builtin_contexts +from powercontext.builtin.runtime import ( + BuiltinConfig, + SubmitSourceObservation, + open_builtin_contexts, + open_builtin_runtime, +) from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration @@ -226,3 +234,45 @@ async def scenario() -> None: ) asyncio.run(scenario()) + + +def test_shared_merge_and_restore_preserve_source_evidence(migration_backend: MigrationBackend, tmp_path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) + + async def scenario() -> None: + async with open_builtin_runtime( + BuiltinConfig(database=migration_backend.config), + source_registry=registry, + scheduler_path=tmp_path / "scheduler.db", + ) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Evidence", summary="Shared merge evidence", idempotency_key="source-evidence") + ) + contexts = cast(RelationalContexts, runtime._provider) + source = SourceRef(source_type="content", source_id="release") + await contexts.records.capture_source(scope.scope_id, "content", "release", "Exact release material.", {}) + for identity in ("a", "b"): + await _commit(contexts, scope.scope_id, identity, identity + " rule.", sources=(source,)) + memory = contexts.atomic_memory.for_scope(scope.scope_id) + accepted = await memory.evidence("a", revision=1) + assert runtime.artifact_merge is not None + merge = runtime.artifact_merge.for_scope(scope.scope_id, "atomic-memory") + a, b = await merge.get("a"), await merge.get("b") + await merge.merge( + (a.as_read(), b.as_read()), + AtomicMemoryContent(kind="fact", text="Combined release rule."), + artifact_id="combined", + lineage=ArtifactLineage(sources=(source,), artifacts=(a.ref, b.ref)), + ) + annotation = await memory.evidence("combined", revision=1) + assert annotation.sources == accepted.sources + assert annotation.sources[0].declaration == declaration + restored = await merge.restore("a") + assert restored.primary.ref.revision == 2 + assert (await memory.evidence("a", revision=2)).sources[0].declaration == declaration + assert await memory.evidence("combined", revision=1) == annotation + assert await memory.evidence("a", revision=1) == accepted + + asyncio.run(scenario()) From 6a836b1106e1056cee5c00f85b27c89939b4c4d8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 12:05:05 +0800 Subject: [PATCH 11/26] fix(sqlite): preserve startup catalog compatibility --- .../builtin/persistence/migrations/atomic_memory_evidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py index 14e2e2809f..d23f81fd2e 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -105,7 +105,7 @@ async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tab async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: if dialect == "sqlite": actual = await connection.exec_driver_sql( - "SELECT type, name, sql FROM sqlite_schema WHERE (name = ? OR tbl_name = ?) " + "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", (name, name), ) From b5a4c1e38534f8ecfe4842de945d9331932867f3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 12:05:05 +0800 Subject: [PATCH 12/26] fix(sources): keep unverifiable evidence neutral Check native serialization compatibility at acceptance without blocking valid captures or Memory writes. Keep previously accepted incompatible declarations neutral while preserving exact material and digest checks. Reject legacy identities that cannot be decoded exactly instead of supplying a default Definition version. --- .../builtin/persistence/sources.py | 91 ++++++++++++------- tests/builtin/persistence/test_sources.py | 3 + .../e2e/test_atomic_memory_source_evidence.py | 69 +++++++++++++- 3 files changed, 129 insertions(+), 34 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 24cff31caf..e2c465ad2e 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -307,7 +307,9 @@ async def historical_evidence( row = by_ref.get(ref) if row is None: raise RepositoryNotFoundError("source", (scope_id, reference)) - source, material = await self._historical_material(connection, reference, row) + source, material, verifiable = await self._historical_material(connection, reference, row) + if not verifiable: + fingerprints[ref] = "" # Never authorize a retained declaration for this capture. if isinstance(source, SourceObservation): fingerprints[ref] = source.definition_fingerprint elif ref[0] == CONTENT_SOURCE_NAME: @@ -334,33 +336,14 @@ async def historical_evidence( async def _historical_material( self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] - ) -> tuple[Source, dict[str, JsonValue]]: + ) -> tuple[Source, dict[str, JsonValue], bool]: """Validate the stored representation and its retained schema, not a live model.""" payload = stored_bytes(row["payload"], column="payload") try: envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) except InvalidStoredPayloadError: - # Bare legacy material has no representation discriminator. Use - # its registered decoder when available; otherwise retain opaque - # canonical fields for neutral maintenance, not a guessed schema. - try: - self._registry.definition_for_name(reference.source_type) - except SourceDefinitionNotFoundError: - source = load_model(Source, payload, kind="source", name=reference.source_type) - if source.name != reference.source_id: - raise IdentityMismatchError("source", reference.source_id, source.name) from None - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error - return source, material - source = self._decode_row(row).value - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error - return source, material + return self._historical_legacy_material(reference, row, payload) material = envelope.value source = self._historical_source(reference, envelope) decoded = SourceRef( @@ -370,22 +353,30 @@ async def _historical_material( if reference != decoded: raise IdentityMismatchError("source", reference, decoded) table = SOURCE_EVIDENCE_BINDINGS_TABLE - fingerprint = await connection.scalar( - select(table.c.manifest_fingerprint).where( - table.c.scope_id == row["scope_id"], - table.c.source_type == reference.source_type, - table.c.source_id == reference.source_id, - table.c.definition_name == reference.source_type, - table.c.definition_version == source.definition_version, + binding = ( + await connection.execute( + select(table.c.manifest_fingerprint, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) ) - ) + ).one_or_none() manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) - if manifest is not None and manifest.fingerprint == fingerprint: + if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: try: Draft202012Validator.check_schema(manifest.source_schema) value = source.payload if isinstance(source, SourceObservation) else material Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) except JsonSchemaValidationError as error: + if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( + material + ): + # Older releases bound declarations without checking serialization + # compatibility. Exact accepted bytes remain readable, but neutral. + return source, material, False raise InvalidStoredPayloadError( "source", reference.source_type, "payload does not match the historical schema" ) from error @@ -393,7 +384,29 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) else: self._validate_unbound_material(reference, envelope, source) - return source, material + return source, material, True + + def _historical_legacy_material( + self, reference: SourceRef, row: Mapping[Any, Any], payload: bytes + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Read bare legacy material without guessing absent identity fields.""" + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if not {"name", "definition_version", "materialization"} <= source.model_fields_set: + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) from None + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + else: + source = self._decode_row(row).value + return source, material, True def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: """Decode identity independently of retained material and today's business defaults.""" @@ -423,6 +436,10 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) header = _normalize_native_material(definition.source_class, material) + if not {"name", "definition_version", "materialization"} <= header.keys(): + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) try: return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) except InvalidStoredPayloadError: @@ -526,6 +543,16 @@ async def _bind_memory_evidence( else: definition = self._registry.definition_for_source(source) manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + if manifest is not None: + # A native decoder can accept representations its generated schema + # cannot describe. Such captures are valid, but cannot attest evidence. + try: + Draft202012Validator.check_schema(manifest.source_schema) + Draft202012Validator(manifest.source_schema, registry=Registry()).validate( + source.model_dump(mode="json", by_alias=True) + ) + except (JsonSchemaValidationError, SchemaError, Unresolvable): + manifest = None declaration = ( MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index cb1e0a5521..9df98d87d2 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -25,6 +25,7 @@ from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) @@ -297,6 +298,8 @@ async def scenario() -> None: ) ) ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(InvalidStoredPayloadError): + await SourceRepository(()).historical_evidence(connection, "scope-a", (ref,)) loaded = await repository.get(connection, "scope-a", ref) assert loaded.value == source assert await repository.add(connection, "scope-a", source) == loaded diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 7e0178c5df..551175e8c3 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -22,6 +22,7 @@ from typing import cast import pytest +from pydantic import field_serializer, field_validator from sqlalchemy import event, text from sqlalchemy.exc import SQLAlchemyError @@ -42,10 +43,10 @@ from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION -from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceMaterialization, SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.observations import manifest_for_definition, project_source_for_transport -from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource from tests.e2e.atomic_memory_migration_backend import MigrationBackend from tests.e2e.dream_support import atomic_memory_pipeline @@ -70,6 +71,70 @@ async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), ar return result.primary +@pytest.mark.parametrize("declared", [False, True]) +def test_unverifiable_native_serializer_keeps_memory_evidence_neutral(migration_backend, declared) -> None: + class SerializedNote(NoteSource): + count: int = 12 + + @field_serializer("count") + def serialize_count(self, value): + return str(value) + + @field_validator("count", mode="before") + @classmethod + def parse_count(cls, value): + return int(value) + + class SerializedAdapter(NoteAdapter): + source_class = SerializedNote + + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializedAdapter(), memory_evidence=declaration if declared else None), + )) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario(): + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = SerializedNote(name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained.") + await context.sources.add(source) + assert await context.sources.get(source) == source + assert await context.sources.add(source) == source + ref = SourceRef(source_type="note", source_id=source.name) + record = await _commit(contexts, "scope", "serialized", "Retained fact.", sources=(ref,)) + evidence = await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) + assert evidence.artifact_ref == record.ref + assert evidence.sources[0].source_ref == ref + assert evidence.sources[0].declaration == MemoryEvidenceDeclaration() + assert evidence.sources[0].manifest_fingerprint is None + if declared: + # Retain the binding shape accepted before schema compatibility checks. + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "UPDATE pc_source_evidence_bindings SET manifest_fingerprint = " + "(SELECT fingerprint FROM pc_source_definition_manifests WHERE definition_name = 'note'), " + "declaration = :declaration WHERE scope_id = 'scope' AND source_id = 'serialized'" + ), + {"declaration": declaration.model_dump_json().encode()}, + ) + await _commit(contexts, "scope", "legacy", "Previously accepted fact.", sources=(ref,)) + legacy = await contexts.atomic_memory.for_scope("scope").evidence("legacy", revision=1) + assert legacy.sources == evidence.sources + async with open_builtin_contexts(config, source_registry=SourceDefinitionRegistry(())) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + assert await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) == evidence + + asyncio.run(scenario()) + + def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") From 2bce54e6fb33a18dccae4825f0218e887bff20a3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 13:39:43 +0800 Subject: [PATCH 13/26] fix(sources): reject omitted identity before capture Reject native serialized material that omits an identity field before journal allocation or persistence, rather than accepting Sources that later fail evidence capture. Preserve strict retained identity checks. --- .../builtin/persistence/sources.py | 4 ++++ tests/builtin/persistence/test_sources.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index e2c465ad2e..92e0a543a4 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -914,6 +914,10 @@ def _dump_source( source=None if representation == "observation" else _source_header(source), source_keys=None if representation == "observation" else _source_keys(source), ) + if envelope.source_keys is not None: + for key, alias in envelope.source_keys.items(): + if alias not in envelope.value: + raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 9df98d87d2..dcfe399aaf 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -308,6 +308,26 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_excluded_native_identity_is_rejected_before_capture() -> None: + class ExcludedVersionSource(NativeCamelCaseSource): + definition_version: str = Field(default="1", exclude=True) + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(ExcludedVersionSource),)) + source = ExcludedVersionSource( + name="excluded", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: class AliasedSource(NativeCamelCaseSource): name: str = Field(alias="source_name") From bb36c937d43f99011b516fb74e4d984e1562c783 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 17:36:26 +0800 Subject: [PATCH 14/26] fix(sources): reject transformed identity before capture Validate serialized identity values as well as key presence before persistence, matching historical evidence validation. Cover round-trippable codecs for all three identity fields without weakening retained-material checks. --- .../builtin/persistence/sources.py | 2 ++ tests/builtin/persistence/test_sources.py | 34 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 92e0a543a4..726e249e29 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -918,6 +918,8 @@ def _dump_source( for key, alias in envelope.source_keys.items(): if alias not in envelope.value: raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") + if envelope.value[alias] != getattr(source, key): + raise InvalidStoredPayloadError("source", name, f"serialized native identity changes {key!r}") return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index dcfe399aaf..6c4802af34 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -16,9 +16,10 @@ import asyncio import json +from urllib.parse import quote, unquote import pytest -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel from sqlalchemy import insert @@ -328,6 +329,37 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("identity_field", ["name", "definition_version", "materialization"]) +def test_encoded_native_identity_is_rejected_before_capture(identity_field) -> None: + class EncodedSource(NativeCamelCaseSource): + @field_serializer("name", "definition_version", "materialization") + def encode_identity(self, value, info): + if info.field_name == identity_field: + return quote(f"wire:{value}", safe="") + return value + + @field_validator("name", "definition_version", "materialization", mode="before") + @classmethod + def decode_identity(cls, value, info): + if info.field_name == identity_field: + return unquote(value).removeprefix("wire:") + return value + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(EncodedSource),)) + source = EncodedSource(name="note/42", materialization=SourceMaterialization.CAPTURED, body="Retained.") + assert EncodedSource.model_validate_json(source.model_dump_json(by_alias=True)) == source + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: class AliasedSource(NativeCamelCaseSource): name: str = Field(alias="source_name") From 73ec5aa8348ac642da87711207aa39ac9dd5a6c8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 13:08:43 +0800 Subject: [PATCH 15/26] fix(sources): separate worker and historical identity decoding Canonicalize worker envelope keys and their manifest without changing native persisted schemas. Read retained canonical identities before consulting current adapter aliases, preserving neutral historical evidence. --- .../builtin/persistence/source_definitions.py | 5 +- .../builtin/persistence/sources.py | 12 ++- src/powercontext/sources/observations.py | 48 +++++++++- tests/builtin/persistence/test_sources.py | 74 ++++++++++++++- tests/test_source_observations.py | 90 ++++++++++++++++++- 5 files changed, 221 insertions(+), 8 deletions(-) diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 3705e9b566..beffb3f87f 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -32,7 +32,8 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinition, SourceDefinitionManifest, manifest_for_definition +from powercontext.sources import SourceDefinition, SourceDefinitionManifest +from powercontext.sources.observations import _manifest_for_definition class SourceDefinitionManifestRepository: @@ -86,7 +87,7 @@ async def register_native( """ try: - manifest = manifest_for_definition(definition) + manifest = _manifest_for_definition(definition, transport=False) except Exception: # Native schema providers may not expose a valid manifest. return None try: diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 726e249e29..d896200386 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -422,6 +422,10 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa raise IdentityMismatchError("source-envelope", value, material.get(alias)) return envelope.source # Legacy aliased rows need their registered decoder once; canonical rows do not. + if {"name", "definition_version", "materialization"} <= material.keys() and material[ + "name" + ] == reference.source_id: + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) if any( @@ -430,8 +434,6 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa ): source, _ = self._decode_envelope(reference.source_type, envelope) return source - if {"name", "definition_version", "materialization"} <= material.keys(): - return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) header = material with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) @@ -459,6 +461,12 @@ def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSour definition = self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: return + if envelope.source is None and any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + # Today's identity aliases cannot validate an older unbound representation. + return if definition.version == source.definition_version: self._decode_envelope(reference.source_type, envelope) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index c9bb8a802f..f3e11e676c 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -161,9 +161,26 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build the immutable declaration transported by a remote worker.""" + """Build a worker declaration with canonical envelope keys and aliased business fields.""" + return _manifest_for_definition(definition, transport=True) + + +def _manifest_for_definition( + definition: SourceDefinition[Any, Any, Any], + /, + *, + transport: bool, +) -> SourceDefinitionManifest: + aliases = _transport_identity_aliases(definition.source_class) if transport else {} source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) + if aliases: + properties = source_schema.get("properties") + if isinstance(properties, dict): + source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} + required = source_schema.get("required") + if isinstance(required, list): + source_schema["required"] = [aliases.get(key, key) if isinstance(key, str) else key for key in required] projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -200,6 +217,8 @@ def project_source_for_transport( ) manifest = manifest_for_definition(definition) payload = _json_object(source.model_dump(mode="json", by_alias=True)) + aliases = _transport_identity_aliases(type(source)) + payload = {aliases.get(key, key): value for key, value in payload.items()} projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) @@ -215,6 +234,33 @@ def project_source_for_transport( ) +def _transport_identity_aliases(source_class: type[Source], /) -> dict[str, str]: + """Reject ambiguous aliases before moving envelope keys in schemas or values.""" + + identity = Source.model_fields + fields = source_class.model_fields + serialized: dict[str, str] = {} + projected: set[str] = set() + for name, field in fields.items(): + if field.exclude: + continue + alias = field.serialization_alias if field.serialization_alias is not None else name + if alias in serialized: + raise InvalidSourceDefinitionError( + source_class, + "serialization_alias", + f"{alias!r} is shared by {serialized[alias]!r} and {name!r}", + ) + serialized[alias] = name + target = name if name in identity else alias + if target in projected: + raise InvalidSourceDefinitionError( + source_class, "serialization_alias", f"{target!r} collides after envelope normalization" + ) + projected.add(target) + return {alias: name for alias, name in serialized.items() if name in identity} + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6c4802af34..6bdd8bc39a 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -21,7 +21,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel -from sqlalchemy import insert +from sqlalchemy import delete, insert, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( @@ -30,6 +30,7 @@ RepositoryNotFoundError, StoredPayloadConflictError, ) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import dump_model from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile @@ -269,6 +270,77 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("current_version", ["1", "2"]) +def test_canonical_history_survives_current_identity_aliases(current_version) -> None: + class CurrentSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(NativeCamelCaseSource),)) + source = NativeCamelCaseSource( + name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + current = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version=current_version), + )) + ) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "1" + assert expected[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("business_version", ["1", "2"]) +def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: + class AliasedSource(NativeCamelCaseSource): + definition_version: str = Field(alias="source_version") + materialization: SourceMaterialization = Field(alias="source_materialization") + business_version: str = Field(alias="definition_version") + business_kind: str = Field(alias="materialization") + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(AliasedSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + source = AliasedSource( + name="aliased-history", source_version="2", source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, materialization="referenced", body="Retained.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + payload = json.dumps({ + "encoding": "powercontext-source-v1", "representation": "native", + "value": source.model_dump(mode="json", by_alias=True), + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + assert await repository.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "2" + assert expected[0].materialization is SourceMaterialization.CAPTURED + assert expected[0].declaration == declaration + + asyncio.run(scenario()) + + def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry(( diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index c313ae7962..cf27fe086a 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -18,17 +18,24 @@ import urllib.request import pytest -from pydantic import ValidationError +from pydantic import Field, ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.runtime.relational import _json_schema_validator +from powercontext import InvalidSourceDefinitionError +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import SHARED_TABLES +from powercontext.builtin.runtime import SubmitSourceObservation +from powercontext.builtin.runtime.relational import RelationalContexts, _json_schema_validator from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + AdapterSourceDefinition, Source, SourceCatalog, SourceDefinitionManifest, SourceDefinitionRegistry, + SourceMaterialization, SourceObservation, TextEvidence, manifest_for_definition, @@ -75,6 +82,85 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize( + ("name_alias", "body_alias"), + [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], +) +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion(name_alias: str, body_alias: str) -> None: + class AliasedSource(Source): + name: str = Field(serialization_alias=name_alias) + definition_version: str = Field(default="1", serialization_alias="wire_version") + materialization: SourceMaterialization = Field(serialization_alias="wire_materialization") + description: str | None = Field(default=None, serialization_alias="wire_description") + body: str = Field(serialization_alias=body_alias) + + class AliasedAdapter: + name = "aliased" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Aliased description.", + body=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Retain the aliased business value.") + if body_alias == "name" or name_alias == body_alias: + with pytest.raises(InvalidSourceDefinitionError): + manifest_for_definition(definition) + with pytest.raises(InvalidSourceDefinitionError): + project_source_for_transport(worker, source) + return + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "remote-aliased", + "definition_version": "2", + "materialization": "captured", + "description": "Aliased description.", + "wire_text": "Retain the aliased business value.", + } + assert observation.payload == expected + assert source.model_dump(mode="json", by_alias=True) == { + name_alias: "remote-aliased", + "wire_version": "2", + "wire_materialization": "captured", + "wire_description": "Aliased description.", + "wire_text": "Retain the aliased business value.", + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "remote-aliased" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as native_profile: + async with native_profile.database.transaction() as connection: + native_manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + assert native_manifest is not None + assert native_manifest.source_schema == AliasedSource.model_json_schema(mode="serialization") + assert native_manifest.fingerprint != manifest.fingerprint + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From 25d9e7a7d3514a7110761c472f0efb947b2e5826 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 13:22:07 +0800 Subject: [PATCH 16/26] fix(sources): disambiguate canonical history from business aliases Only trust retained canonical identity keys when the registered model does not claim those spellings for business fields. Keep headerless business-aliased material on the declared decoder, and format the regression file with the pinned hook version. --- .../builtin/persistence/sources.py | 21 ++++++++++++++++++- tests/builtin/persistence/test_sources.py | 11 +++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index d896200386..fa4813985c 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -425,7 +425,11 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa if {"name", "definition_version", "materialization"} <= material.keys() and material[ "name" ] == reference.source_id: - return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + definition = None + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if definition is None or not _claims_identity_keys(definition.source_class): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) if any( @@ -879,6 +883,21 @@ def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) - return keys +def _claims_identity_keys(source_class: type[Source]) -> bool: + """Report whether a business field serializes to a canonical identity spelling. + + A headerless envelope cannot distinguish an identity value from a business + value with that name. When the registered model claims the spelling for a + different field, only its declared aliases are authoritative. + """ + + return any( + name not in Source.model_fields and key in (field.alias, field.validation_alias, field.serialization_alias) + for name, field in source_class.model_fields.items() + for key in Source.model_fields + ) + + def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: """Read pre-canonical inherited envelope aliases, never business-field aliases. diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6bdd8bc39a..849ef7fcdb 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -320,14 +320,19 @@ async def scenario() -> None: )) repository = SourceRepository(registry) source = AliasedSource( - name="aliased-history", source_version="2", source_materialization=SourceMaterialization.CAPTURED, - definition_version=business_version, materialization="referenced", body="Retained.", + name="aliased-history", + source_version="2", + source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, + materialization="referenced", + body="Retained.", ) async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: stored = await repository.add(connection, "scope-a", source) expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) payload = json.dumps({ - "encoding": "powercontext-source-v1", "representation": "native", + "encoding": "powercontext-source-v1", + "representation": "native", "value": source.model_dump(mode="json", by_alias=True), }).encode() await connection.execute( From 5ceed572d1e938cc7f4251cf90060a32fbc555d5 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 21:00:11 +0800 Subject: [PATCH 17/26] fix(sources): normalize recursive transport root schemas Resolve referenced root schemas before canonicalizing transport identity keys while preserving nested serialization aliases. Cover manifest registration, observation submission, and SQLite readback. --- src/powercontext/sources/observations.py | 9 ++++ tests/test_source_observations.py | 65 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index f3e11e676c..981f3d258c 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -175,6 +175,15 @@ def _manifest_for_definition( aliases = _transport_identity_aliases(definition.source_class) if transport else {} source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) if aliases: + reference = source_schema.get("$ref") + definitions = source_schema.get("$defs") + if isinstance(reference, str) and reference.startswith("#/$defs/") and isinstance(definitions, dict): + key = reference.removeprefix("#/$defs/").replace("~1", "/").replace("~0", "~") + root = definitions.get(key) + if isinstance(root, dict): + # Inline only the root: recursive children retain their serialized aliases in $defs. + source_schema = {**source_schema, **root} + source_schema.pop("$ref") properties = source_schema.get("properties") if isinstance(properties, dict): source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index cf27fe086a..a152d352a5 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -16,6 +16,7 @@ import asyncio import urllib.request +from typing import Self import pytest from pydantic import Field, ValidationError @@ -161,6 +162,70 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_recursive_source_transport_preserves_nested_serialization_aliases() -> None: + class RecursiveSource(Source): + name: str = Field(serialization_alias="source_name") + definition_version: str = Field(default="1", serialization_alias="wire_version") + body: str = Field(serialization_alias="wire_text") + children: list[Self] = Field(default_factory=list) + + class RecursiveAdapter: + name = "recursive" + input_class = str + source_class = RecursiveSource + + async def resolve(self, value: str, /) -> RecursiveSource: + leaf = RecursiveSource(name="leaf", materialization=SourceMaterialization.CAPTURED, body="Nested text.") + return RecursiveSource( + name="root", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + children=[leaf], + ) + + async def read(self, source: RecursiveSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(RecursiveAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Root text.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "root", + "definition_version": "2", + "materialization": "captured", + "description": None, + "wire_text": "Root text.", + "children": [ + { + "source_name": "leaf", + "wire_version": "1", + "materialization": "captured", + "description": None, + "wire_text": "Nested text.", + "children": [], + } + ], + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "root" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From 2b95df67139460e76a0aaf35162d3a0ff97dfc53 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 21 Oct 2026 11:16:59 +0800 Subject: [PATCH 18/26] fix(sources): preserve versioned alias compatibility Keep remote manifests and payloads on the canonical field-name contract so existing Definition fingerprints remain stable. Preserve native historical identity and retained alias spellings across upgrades, replay, and rebuild. --- .../builtin/persistence/sources.py | 137 +++++++++++-- src/powercontext/sources/models.py | 13 +- src/powercontext/sources/observations.py | 56 +----- tests/builtin/persistence/test_sources.py | 185 +++++++++++++++++- .../e2e/test_atomic_memory_source_evidence.py | 69 ++++++- tests/test_source_observations.py | 87 +++++--- 6 files changed, 442 insertions(+), 105 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index fa4813985c..b32990e757 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -25,7 +25,7 @@ from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter +from pydantic import AliasChoices, AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter from referencing import Registry from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update @@ -343,9 +343,15 @@ async def _historical_material( try: envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) except InvalidStoredPayloadError: - return self._historical_legacy_material(reference, row, payload) + return await self._historical_legacy_material(connection, reference, row, payload) material = envelope.value - source = self._historical_source(reference, envelope) + source = ( + await self._historical_canonical_source(connection, reference, row, material) + if envelope.representation == "native" and envelope.source is None + else None + ) + if source is None: + source = self._historical_source(reference, envelope) decoded = SourceRef( source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, source_id=source.name, @@ -386,14 +392,21 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) return source, material, True - def _historical_legacy_material( - self, reference: SourceRef, row: Mapping[Any, Any], payload: bytes + async def _historical_legacy_material( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + payload: bytes, ) -> tuple[Source, dict[str, JsonValue], bool]: """Read bare legacy material without guessing absent identity fields.""" try: material = _JSON_OBJECT.validate_json(payload, strict=True) except ValueError as error: raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + source = await self._historical_canonical_source(connection, reference, row, material) + if source is not None: + return source, material, False try: self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: @@ -408,6 +421,78 @@ def _historical_legacy_material( source = self._decode_row(row).value return source, material, True + async def _historical_canonical_source( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + material: dict[str, JsonValue], + ) -> Source | None: + """Recover evidence identity from canonical retained material, never a live alias.""" + + if not {"name", "definition_version", "materialization"} <= material.keys(): + return None + if material["name"] != reference.source_id: + return None + digest_material = material + if reference.source_type == CONTENT_SOURCE_NAME: + digest_material = {key: value for key, value in material.items() if key != "handoff_receipt"} + material_digest = _source_material_digest_material(digest_material) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.definition_name, table.c.definition_version, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + ) + ) + ).one_or_none() + if binding is not None: + if binding.definition_name != reference.source_type or binding.material_digest != material_digest: + return None + version = binding.definition_version + elif material["definition_version"] == "1": + version = "1" + else: + return None + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + definition = None + if ( + definition is not None + and definition.version == version + and any( + ( + ( + definition.source_class.model_fields[key].alias is not None + and definition.source_class.model_fields[key].alias != key + ) + or ( + definition.source_class.model_fields[key].serialization_alias is not None + and definition.source_class.model_fields[key].serialization_alias != key + ) + or ( + definition.source_class.model_fields[key].validation_alias is not None + and definition.source_class.model_fields[key].validation_alias != key + ) + ) + for key in ("definition_version", "materialization") + ) + ): + # The current exact-version codec is required to disambiguate retained identity keys. + return None + try: + return Source.model_validate({ + "name": reference.source_id, + "definition_version": version, + "materialization": material["materialization"], + "description": material.get("description"), + }) + except (TypeError, ValueError): + return None + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: """Decode identity independently of retained material and today's business defaults.""" @@ -465,6 +550,9 @@ def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSour definition = self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: return + if _claims_identity_keys(definition.source_class): + # The retained canonical identity/header is authoritative; a live business alias may claim it. + return if envelope.source is None and any( (definition.source_class.model_fields[key].serialization_alias or key) != key for key in ("name", "definition_version", "materialization") @@ -868,18 +956,30 @@ def _source_header(source: Source) -> Source: def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: - """Retain declared identity spellings, including pre-canonical generated keys.""" - keys = { - key: type(source).model_fields[key].serialization_alias or key - for key in ("name", "definition_version", "materialization") - } - generator = type(source).model_config.get("alias_generator") - if isinstance(generator, AliasGenerator): - generator = generator.serialization_alias or generator.alias - if material is not None and generator is not None: - for key, alias in keys.items(): - if alias not in material: - keys[key] = generator(key) + """Retain the exact serialized spelling used for each native identity.""" + keys: dict[str, str] = {} + for key in ("name", "definition_version", "materialization"): + field = type(source).model_fields[key] + preferred = field.serialization_alias or key + candidates = [preferred] + if isinstance(field.validation_alias, str): + candidates.append(field.validation_alias) + elif isinstance(field.validation_alias, AliasChoices): + candidates.extend(choice for choice in field.validation_alias.choices if isinstance(choice, str)) + if field.alias is not None: + candidates.append(field.alias) + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is not None: + candidates.append(generator(key)) + if material is not None: + keys[key] = next( + (alias for alias in dict.fromkeys(candidates) if material.get(alias) == getattr(source, key)), + preferred, + ) + else: + keys[key] = preferred return keys @@ -912,7 +1012,8 @@ def _normalize_native_material(source_class: type[Source], material: dict[str, J return material normalized = dict(material) for key in Source.model_fields: - if source_class.model_fields[key].validation_alias != key: + field = source_class.model_fields[key] + if field.validation_alias != key and field.alias_priority != 1: continue alias = generator(key) if any( diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 0c31b8a387..0f1a4a26fe 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -16,7 +16,7 @@ from enum import StrEnum -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, field_validator from powercontext.errors import InvalidSourceReferenceError from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH @@ -60,11 +60,12 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - # Envelope keys stay canonical for validation and serialization, independently of adapter aliases. - name: str = Field(alias="name") - definition_version: str = Field(default="1", alias="definition_version") - materialization: SourceMaterialization = Field(alias="materialization") - description: str | None = Field(default=None, alias="description") + model_config = ConfigDict(populate_by_name=True) + + name: str + definition_version: str = "1" + materialization: SourceMaterialization + description: str | None = None @field_validator("definition_version") @classmethod diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 981f3d258c..8c1b0c630e 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -161,7 +161,7 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build a worker declaration with canonical envelope keys and aliased business fields.""" + """Build a declaration matching the canonical remote payload representation.""" return _manifest_for_definition(definition, transport=True) @@ -172,24 +172,11 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - aliases = _transport_identity_aliases(definition.source_class) if transport else {} - source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) - if aliases: - reference = source_schema.get("$ref") - definitions = source_schema.get("$defs") - if isinstance(reference, str) and reference.startswith("#/$defs/") and isinstance(definitions, dict): - key = reference.removeprefix("#/$defs/").replace("~1", "/").replace("~0", "~") - root = definitions.get(key) - if isinstance(root, dict): - # Inline only the root: recursive children retain their serialized aliases in $defs. - source_schema = {**source_schema, **root} - source_schema.pop("$ref") - properties = source_schema.get("properties") - if isinstance(properties, dict): - source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} - required = source_schema.get("required") - if isinstance(required, list): - source_schema["required"] = [aliases.get(key, key) if isinstance(key, str) else key for key in required] + source_schema = _json_object( + definition.source_class.model_json_schema(mode="validation", by_alias=False) + if transport + else definition.source_class.model_json_schema(mode="serialization") + ) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -225,9 +212,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json", by_alias=True)) - aliases = _transport_identity_aliases(type(source)) - payload = {aliases.get(key, key): value for key, value in payload.items()} + payload = _json_object(source.model_dump(mode="json", by_alias=False)) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) @@ -243,33 +228,6 @@ def project_source_for_transport( ) -def _transport_identity_aliases(source_class: type[Source], /) -> dict[str, str]: - """Reject ambiguous aliases before moving envelope keys in schemas or values.""" - - identity = Source.model_fields - fields = source_class.model_fields - serialized: dict[str, str] = {} - projected: set[str] = set() - for name, field in fields.items(): - if field.exclude: - continue - alias = field.serialization_alias if field.serialization_alias is not None else name - if alias in serialized: - raise InvalidSourceDefinitionError( - source_class, - "serialization_alias", - f"{alias!r} is shared by {serialized[alias]!r} and {name!r}", - ) - serialized[alias] = name - target = name if name in identity else alias - if target in projected: - raise InvalidSourceDefinitionError( - source_class, "serialization_alias", f"{target!r} collides after envelope normalization" - ) - projected.add(target) - return {alias: name for alias, name in serialized.items() if name in identity} - - def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 849ef7fcdb..6c901725a3 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -21,7 +21,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel -from sqlalchemy import delete, insert, update +from sqlalchemy import delete, insert, select, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( @@ -37,6 +37,7 @@ from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.sources.content import CONTENT_SOURCE_ADAPTER, ContentSource from powercontext.sources import ( AdapterSourceDefinition, MemoryEvidenceDeclaration, @@ -230,7 +231,7 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": "Remote alias description.", - "wire_text": "Remote note text.", + "body": "Remote note text.", } asyncio.run(scenario()) @@ -272,12 +273,13 @@ async def scenario() -> None: @pytest.mark.parametrize("current_version", ["1", "2"]) def test_canonical_history_survives_current_identity_aliases(current_version) -> None: - class CurrentSource(NativeCamelCaseSource): + class CurrentSource(SerializationAliasSource): name: str = Field(alias="source_name") + label: str = Field(alias="name") async def scenario() -> None: - original = SourceRepository((SerializationAliasAdapter(NativeCamelCaseSource),)) - source = NativeCamelCaseSource( + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + source = SerializationAliasSource( name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." ) material = source.model_dump(mode="json", by_alias=True) @@ -305,6 +307,50 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("has_header", [False, True]) +def test_business_name_alias_does_not_reinterpret_canonical_history(has_header: bool) -> None: + class CurrentSource(SerializationAliasSource): + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + current = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="1"),)) + ) + source = SerializationAliasSource( + name="canonical-business-alias", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + material = source.model_dump(mode="json", by_alias=True) + value = {"encoding": "powercontext-source-v1", "representation": "native", "value": material} + if has_header: + value.update( + source=Source( + name=source.name, + definition_version=source.definition_version, + materialization=source.materialization, + ).model_dump(mode="json", exclude_none=True), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE) + .where(SOURCES_TABLE.c.source_id == source.name) + .values(payload=json.dumps(value).encode()) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + + asyncio.run(scenario()) + + @pytest.mark.parametrize("business_version", ["1", "2"]) def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: class AliasedSource(NativeCamelCaseSource): @@ -346,6 +392,33 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: + async def scenario() -> None: + repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) + source = ContentSource( + name="receipt-digest", + materialization=SourceMaterialization.CAPTURED, + content="Receipt content.", + handoff_receipt=True, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + original = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + restored = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + assert restored == original + + asyncio.run(scenario()) + + def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry(( @@ -356,7 +429,6 @@ async def scenario() -> None: name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." ) material = source.model_dump(mode="json", by_alias=True) - material["definitionVersion"] = material.pop("definition_version") payload = json.dumps({ "encoding": "powercontext-source-v1", "representation": "native", @@ -487,6 +559,65 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("representation", ["enveloped", "bare"]) +def test_replay_retains_historical_native_identity_spelling(representation) -> None: + from pydantic import AliasChoices + + class OldSource(NativeCamelCaseSource): + name: str = Field(alias="legacy_name") + + class CurrentSource(NativeCamelCaseSource): + name: str = Field( + validation_alias=AliasChoices("source_name", "legacy_name"), + serialization_alias="source_name", + ) + + async def scenario() -> None: + original_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(OldSource), version="2"), + )) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="2"), + )) + original = SourceRepository(original_registry) + current = SourceRepository(current_registry) + source = OldSource( + legacy_name="legacy-replay", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained exactly.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + material = source.model_dump(mode="json", by_alias=True) + if representation == "bare": + payload = json.dumps(material).encode() + else: + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + loaded = await current.get(connection, "scope-a", stored.ref) + assert loaded.value.name == source.name + replay = await current.add(connection, "scope-a", loaded.value) + assert replay.ref == stored.ref + assert await current.get(connection, "scope-a", stored.ref) == loaded + evidence = await current.historical_evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].source_ref == stored.ref + raw = await connection.scalar( + select(SOURCES_TABLE.c.payload).where(SOURCES_TABLE.c.source_id == source.name) + ) + assert isinstance(raw, bytes) + envelope = json.loads(raw) + assert envelope["source_keys"]["name"] == "legacy_name" + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -649,3 +780,45 @@ async def add(scope_id: str, source: NoteSource): assert following.journal_position == 2 asyncio.run(scenario()) + + +def test_nested_uppercase_source_aliases_survive_read_replay_and_restart(tmp_path) -> None: + class UpperNested(Source): + model_config = ConfigDict(alias_generator=str.upper) + body: str + + class NestedNativeSource(NativeCamelCaseSource): + nested: UpperNested + + async def scenario() -> None: + adapter = SerializationAliasAdapter(NestedNativeSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + repository = SourceRepository(registry) + source = NestedNativeSource( + name="nested-legacy", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Outer.", + nested=UpperNested( + name="inner", + materialization=SourceMaterialization.REFERENCED, + body="Nested.", + ), + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nested-alias.db'}") + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + original = await repository.add(connection, "scope-a", source) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + loaded = await repository.get(connection, "scope-a", original.ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == original + evidence = await repository.historical_evidence(connection, "scope-a", (original.ref,)) + assert evidence[0].source_ref == original.ref + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 551175e8c3..928fca8780 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -22,8 +22,8 @@ from typing import cast import pytest -from pydantic import field_serializer, field_validator -from sqlalchemy import event, text +from pydantic import Field, field_serializer, field_validator +from sqlalchemy import event, text, update from sqlalchemy.exc import SQLAlchemyError from powercontext.artifacts import ArtifactLineage @@ -33,6 +33,7 @@ AtomicMemoryExtractionOutput, ) from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.persistence.tables import SOURCES_TABLE from powercontext.builtin.runtime import ( BuiltinConfig, SubmitSourceObservation, @@ -135,6 +136,70 @@ async def scenario(): asyncio.run(scenario()) +@pytest.mark.parametrize("has_header", [False, True]) +def test_headerless_canonical_v1_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool +) -> None: + class CurrentNote(NoteSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + class CurrentAdapter(NoteAdapter): + source_class = CurrentNote + + config = BuiltinConfig(database=migration_backend.config) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) + current_registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="2"),)) + + async def scenario(): + source = NoteSource(name="canonical-v1", materialization=SourceMaterialization.CAPTURED, body="Retained.") + ref = SourceRef(source_type="note", source_id=source.name) + async with open_builtin_contexts(config, source_registry=old_registry) as contexts: + context = await contexts.get("scope") + await context.sources.add(source) + await _commit(contexts, "scope", "canonical", "Canonical historical fact.", sources=(ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload_value = { + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + } + if has_header: + payload_value.update( + source=source.model_dump( + mode="json", + by_alias=False, + include={"name", "definition_version", "materialization"}, + ), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + payload = json.dumps(payload_value).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + + async with open_builtin_contexts(config, source_registry=current_registry) as contexts: + before = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert before.sources[0].source_ref == ref + assert before.sources[0].declaration == MemoryEvidenceDeclaration() + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + after = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert after == before + + asyncio.run(scenario()) + + def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index a152d352a5..0fab312e44 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,14 +15,15 @@ from __future__ import annotations import asyncio +import hashlib import urllib.request from typing import Self import pytest -from pydantic import Field, ValidationError +import rfc8785 +from pydantic import Field, ValidationError, field_serializer from referencing.exceptions import Unresolvable -from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES @@ -87,7 +88,9 @@ async def scenario() -> None: ("name_alias", "body_alias"), [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], ) -def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion(name_alias: str, body_alias: str) -> None: +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion( + name_alias: str, body_alias: str, tmp_path +) -> None: class AliasedSource(Source): name: str = Field(serialization_alias=name_alias) definition_version: str = Field(default="1", serialization_alias="wire_version") @@ -116,12 +119,6 @@ async def scenario() -> None: definition = AdapterSourceDefinition(AliasedAdapter(), version="2") worker = SourceDefinitionRegistry((definition,)) source = await worker.resolve("Retain the aliased business value.") - if body_alias == "name" or name_alias == body_alias: - with pytest.raises(InvalidSourceDefinitionError): - manifest_for_definition(definition) - with pytest.raises(InvalidSourceDefinitionError): - project_source_for_transport(worker, source) - return manifest = manifest_for_definition(definition) observation = project_source_for_transport(worker, source) expected = { @@ -129,20 +126,28 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": "Aliased description.", - "wire_text": "Retain the aliased business value.", + "body": "Retain the aliased business value.", } - assert observation.payload == expected - assert source.model_dump(mode="json", by_alias=True) == { - name_alias: "remote-aliased", - "wire_version": "2", - "wire_materialization": "captured", - "wire_description": "Aliased description.", - "wire_text": "Retain the aliased business value.", + assert manifest.source_schema == AliasedSource.model_json_schema() + baseline_declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": AliasedSource.model_json_schema(), + "projections": [], } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline_declaration)).hexdigest()}" + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload == expected + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-manifest.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest - async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + current_manifest = manifest_for_definition(definition) + assert current_manifest == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - await contexts.register_source_definition(manifest) + assert await contexts.register_source_definition(current_manifest) == manifest receipt = await contexts.submit_source_observation( SubmitSourceObservation(scope_id="scope-a", observation=observation) ) @@ -162,7 +167,7 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_recursive_source_transport_preserves_nested_serialization_aliases() -> None: +def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") definition_version: str = Field(default="1", serialization_alias="wire_version") @@ -198,14 +203,14 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": None, - "wire_text": "Root text.", + "body": "Root text.", "children": [ { - "source_name": "leaf", - "wire_version": "1", + "name": "leaf", + "definition_version": "1", "materialization": "captured", "description": None, - "wire_text": "Nested text.", + "body": "Nested text.", "children": [], } ], @@ -238,6 +243,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_identity_exclusions_and_serializers_fail_observation_validation() -> None: + class ExcludedIdentitySource(Source): + definition_version: str = Field(default="1", exclude=True) + + class TransformedIdentitySource(Source): + @field_serializer("name") + def encode_name(self, value: str) -> str: + return f"wire:{value}" + + class Adapter: + input_class = str + name = "remote-identity" + + def __init__(self, source_class: type[Source]) -> None: + self.source_class = source_class + + async def resolve(self, value: str, /) -> Source: + return self.source_class(name=value, materialization=SourceMaterialization.CAPTURED) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + for source_class in (ExcludedIdentitySource, TransformedIdentitySource): + definition = AdapterSourceDefinition(Adapter(source_class), version="1") + registry = SourceDefinitionRegistry((definition,)) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema(by_alias=False) + source = await registry.resolve("identity") + with pytest.raises(ValidationError): + project_source_for_transport(registry, source) + + asyncio.run(scenario()) + + def test_remote_schema_references_are_rejected_without_network_access(monkeypatch: pytest.MonkeyPatch) -> None: attempted = False From 13517fdd9c6a4a67ff870a333e161d878460db85 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 21 Oct 2026 20:34:46 +0800 Subject: [PATCH 19/26] fix(sources): keep upgraded alias definitions compatible Generate remote transport schemas with the same alias policy as their payloads so existing serialize_by_alias Definitions keep their fingerprint and observations after upgrade. Recover explicit canonical historical versions beyond version 1 instead of handing accepted material to a newer aliased Definition. --- .../builtin/persistence/sources.py | 12 +++- src/powercontext/sources/observations.py | 16 +++-- tests/builtin/persistence/test_sources.py | 52 ++++++++++++++++ .../e2e/test_atomic_memory_source_evidence.py | 18 ++++-- tests/test_source_observations.py | 61 ++++++++++++++++++- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index b32990e757..88ff8e331e 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -449,11 +449,17 @@ async def _historical_canonical_source( ) ).one_or_none() if binding is not None: - if binding.definition_name != reference.source_type or binding.material_digest != material_digest: + if ( + binding.definition_name != reference.source_type + or binding.material_digest != material_digest + or binding.definition_version != material["definition_version"] + ): return None version = binding.definition_version - elif material["definition_version"] == "1": - version = "1" + elif isinstance(material["definition_version"], str): + # Canonical retained identity is explicit even when its historical + # evidence binding predates the evidence tables. It stays neutral. + version = material["definition_version"] else: return None try: diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 8c1b0c630e..3531a6ab81 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -172,11 +172,15 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - source_schema = _json_object( - definition.source_class.model_json_schema(mode="validation", by_alias=False) - if transport - else definition.source_class.model_json_schema(mode="serialization") - ) + if transport: + source_schema = _json_object( + definition.source_class.model_json_schema( + mode="serialization", + by_alias=bool(definition.source_class.model_config.get("serialize_by_alias")), + ) + ) + else: + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -212,7 +216,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json", by_alias=False)) + payload = _json_object(source.model_dump(mode="json")) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6c901725a3..86f86f4f64 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -392,6 +392,58 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_unbound_versioned_canonical_history_ignores_new_definition_aliases() -> None: + class CurrentSource(SerializationAliasSource): + model_config = ConfigDict(populate_by_name=True) + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + old_source = SerializationAliasSource( + name="old-version-two", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained without an evidence binding.", + ) + old_repository = SourceRepository(()) + + class CurrentAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = CurrentSource + + async def resolve(self, value: SerializationAliasInput, /) -> CurrentSource: + raise AssertionError # Historical evidence must not invoke the current decoder. + + async def read(self, source: CurrentSource, /) -> str: + return source.body + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="3"),)) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + # Model a canonical Source accepted before evidence bindings existed. + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=old_source.name, + payload=json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": old_source.model_dump(mode="json", by_alias=False), + }).encode(), + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=old_source.name) + baseline = await old_repository.historical_evidence(connection, "scope-a", (ref,)) + upgraded = await SourceRepository(registry).historical_evidence(connection, "scope-a", (ref,)) + assert upgraded == baseline + assert upgraded[0].definition_version == "2" + assert upgraded[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: async def scenario() -> None: repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 928fca8780..5d674241d1 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -137,8 +137,9 @@ async def scenario(): @pytest.mark.parametrize("has_header", [False, True]) -def test_headerless_canonical_v1_evidence_survives_business_alias_rebuild( - migration_backend: MigrationBackend, has_header: bool +@pytest.mark.parametrize("historical_version", ["1", "2"]) +def test_canonical_historical_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool, historical_version: str ) -> None: class CurrentNote(NoteSource): name: str = Field(alias="source_name") @@ -148,11 +149,18 @@ class CurrentAdapter(NoteAdapter): source_class = CurrentNote config = BuiltinConfig(database=migration_backend.config) - old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) - current_registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="2"),)) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), version=historical_version),)) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(CurrentAdapter(), version=str(int(historical_version) + 1)), + )) async def scenario(): - source = NoteSource(name="canonical-v1", materialization=SourceMaterialization.CAPTURED, body="Retained.") + source = NoteSource( + name=f"canonical-v{historical_version}", + definition_version=historical_version, + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) ref = SourceRef(source_type="note", source_id=source.name) async with open_builtin_contexts(config, source_registry=old_registry) as contexts: context = await contexts.get("scope") diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 0fab312e44..93cb1b7f52 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -21,7 +21,7 @@ import pytest import rfc8785 -from pydantic import Field, ValidationError, field_serializer +from pydantic import ConfigDict, Field, ValidationError, field_serializer from referencing.exceptions import Unresolvable from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository @@ -167,6 +167,61 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_alias_serialization_remote_manifest_and_payload_remain_upgrade_stable(tmp_path) -> None: + class AliasedSource(Source): + model_config = ConfigDict(serialize_by_alias=True, populate_by_name=True) + body: str = Field(alias="wire_text") + + class AliasedAdapter: + name = "aliased-upgrade" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + wire_text=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained alias.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + expected = AliasedSource.model_json_schema() + expected_fingerprint = hashlib.sha256( + rfc8785.dumps({ + "name": definition.name, + "version": definition.version, + "source_schema": expected, + "projections": [], + }) + ).hexdigest() + assert manifest.source_schema == expected + assert manifest.fingerprint == f"sha256:{expected_fingerprint}" + assert observation.payload["wire_text"] == "Retained alias." + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-alias-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "remote-aliased" + + asyncio.run(scenario()) + + def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") @@ -269,7 +324,9 @@ async def scenario() -> None: for source_class in (ExcludedIdentitySource, TransformedIdentitySource): definition = AdapterSourceDefinition(Adapter(source_class), version="1") registry = SourceDefinitionRegistry((definition,)) - assert manifest_for_definition(definition).source_schema == source_class.model_json_schema(by_alias=False) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema( + mode="serialization", by_alias=False + ) source = await registry.resolve("identity") with pytest.raises(ValidationError): project_source_for_transport(registry, source) From 3a8b38be1037672010f0bba1597060a9994c4dd3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 22 Oct 2026 16:39:41 +0800 Subject: [PATCH 20/26] fix(sources): preserve stable remote manifest fingerprints Keep the pre-upgrade validation schema unless aliases rename a required payload key, so unchanged Definitions with Decimal or optional aliased fields retain their registered fingerprints while generated aliases still emit a matching schema. --- src/powercontext/sources/observations.py | 66 +++++++++++++- tests/test_source_observations.py | 110 ++++++++++++++++++++++- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 3531a6ab81..596eff9b71 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,10 +17,20 @@ from __future__ import annotations import hashlib -from typing import Any, Literal +from typing import Any, Literal, get_args, get_origin import rfc8785 -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator +from pydantic import ( + AliasChoices, + AliasPath, + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + field_validator, + model_validator, +) from powercontext.errors import ( InvalidSourceDefinitionError, @@ -172,7 +182,9 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - if transport: + if transport and _validation_schema_matches_payload(definition.source_class): + source_schema = _json_object(definition.source_class.model_json_schema()) + elif transport: source_schema = _json_object( definition.source_class.model_json_schema( mode="serialization", @@ -232,6 +244,54 @@ def project_source_for_transport( ) +def _validation_schema_matches_payload( + source_class: type[BaseModel], + seen: set[type[BaseModel]] | None = None, +) -> bool: + """Return whether validation-mode keys already describe the serialized payload.""" + + visited = set() if seen is None else seen + if source_class in visited: + return True + visited.add(source_class) + serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) + for name, field in source_class.model_fields.items(): + if field.exclude: + continue + serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name + if field.is_required() and _validation_schema_key(name, field) != serialization_key: + return False + for nested in _nested_models(field.annotation): + if not _validation_schema_matches_payload(nested, visited): + return False + return True + + +def _validation_schema_key(name: str, field: Any) -> str | None: + alias = field.validation_alias + if isinstance(alias, str): + return alias + if isinstance(alias, AliasChoices): + return next((choice for choice in alias.choices if isinstance(choice, str)), None) + if isinstance(alias, AliasPath): + return name # Pydantic keeps the field name for path-based validation. + if alias is not None: + return None + return field.alias if field.alias is not None else name + + +def _nested_models(annotation: object) -> tuple[type[BaseModel], ...]: + origin = get_origin(annotation) + if origin is None: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return (annotation,) + return () + nested: list[type[BaseModel]] = [] + for argument in get_args(annotation): + nested.extend(_nested_models(argument)) + return tuple(nested) + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 93cb1b7f52..cf3ff5cee0 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -17,11 +17,13 @@ import asyncio import hashlib import urllib.request +from decimal import Decimal from typing import Self import pytest import rfc8785 from pydantic import ConfigDict, Field, ValidationError, field_serializer +from pydantic.alias_generators import to_camel from referencing.exceptions import Unresolvable from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository @@ -222,6 +224,110 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: + class VersionedSource(Source): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + body: str + + class VersionedAdapter: + name = "versioned-source" + input_class = str + source_class = VersionedSource + + async def resolve(self, value: str, /) -> VersionedSource: + return VersionedSource( + name="versioned", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + ) + + async def read(self, source: VersionedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(VersionedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": VersionedSource.model_json_schema(), + "projections": [], + } + assert manifest.source_schema == VersionedSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["definition_version"] == "2" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'optional-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "versioned" + + asyncio.run(scenario()) + + +def test_decimal_source_manifest_fingerprint_survives_upgrade(tmp_path) -> None: + class AmountSource(Source): + amount: Decimal + + class AmountAdapter: + name = "amount-source" + input_class = str + source_class = AmountSource + + async def resolve(self, value: str, /) -> AmountSource: + return AmountSource( + name="amount", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + amount=Decimal(value), + ) + + async def read(self, source: AmountSource, /) -> str: + return str(source.amount) + + async def scenario() -> None: + definition = AdapterSourceDefinition(AmountAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("1.25") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": AmountSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert manifest.source_schema == AmountSource.model_json_schema() + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload["amount"] == "1.25" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'decimal-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "amount" + + asyncio.run(scenario()) + + def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") @@ -324,9 +430,7 @@ async def scenario() -> None: for source_class in (ExcludedIdentitySource, TransformedIdentitySource): definition = AdapterSourceDefinition(Adapter(source_class), version="1") registry = SourceDefinitionRegistry((definition,)) - assert manifest_for_definition(definition).source_schema == source_class.model_json_schema( - mode="serialization", by_alias=False - ) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema() source = await registry.resolve("identity") with pytest.raises(ValidationError): project_source_for_transport(registry, source) From 95c15361984398a0ae36e1021edb5e2bffe23bfa Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 26 Oct 2026 15:36:27 +0800 Subject: [PATCH 21/26] fix(portability): preserve accepted evidence in portable archives Export and restore now carry the retained evidence rows that were previously dropped silently. A restored scope kept its revisions and lineage but resolved every declaration to neutral, so exporting and restoring a scope lost the accepted evidence permanently. - register `source_evidence_binding` and `atomic_memory_evidence_history` as portable record kinds, with schema and dependency validation - export the Definition manifests referenced by bindings and evidence snapshots, not only those referenced by Source observations - reject an archive whose evidence references a missing manifest instead of publishing one that would discard the declaration The record types are additive under format version 1, which already allows readers to reject unknown record kinds. Docs list the new preserved records in both locales. --- docs/en/docs/operate/portable-archive.md | 1 + docs/zh/docs/operate/portable-archive.md | 1 + .../builtin/portability/bundle.py | 126 ++++++++++++++++-- tests/e2e/test_portable_archive.py | 51 ++++++- 4 files changed, 170 insertions(+), 9 deletions(-) diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index d53274f328..e80626aaa7 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -131,6 +131,7 @@ Format version 1 carries the portable relational representation of these support | --- | --- | | Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | | Source journal heads, Source records, and referenced remote Source Definition manifests | Search projections and indexes | +| Accepted Memory evidence: per-Source declaration bindings, the Definition manifests they reference, and exact-revision evidence snapshots | Today's registry declarations, which never replace a saved snapshot | | Artifact Revisions, their timestamps/actors/rollback metadata, stable handles, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | | Atomic Memory merge relationships and marked merge-input lineage | External Skill registrations and host-local installation state | | Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index 77eeecf320..5bf0cd05d7 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -111,6 +111,7 @@ powercontext archive restore ./backups/payments.pcb --yes | --- | --- | | Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | | Source journal head、Source record 和引用的远程 Source Definition manifest | 搜索 projection 和 index | +| 已接受的 Memory evidence：逐 Source 的 declaration binding、其引用的 Definition manifest 和精确 revision 的 evidence snapshot | 当前 registry 的 declaration，它不会覆盖已保存的快照 | | Artifact Revision 及其时间戳、actor、回滚元数据；稳定句柄、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | | Atomic Memory 合并关系和带合并输入标记的 lineage | External Skill registration 和 host-local installation state | | Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 54bb08b822..892de7b955 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -74,6 +74,10 @@ verdict_key, ) from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) from powercontext.builtin.persistence.candidates import decode_candidate_proposal_envelope from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( @@ -116,6 +120,7 @@ from powercontext.builtin.tags import normalize_tags from powercontext.errors import SourceError from powercontext.sources import SourceDefinitionManifest, SourceObservation +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence FORMAT_VERSION = 1 _RECORDS_NAME = "records.ndjson" @@ -164,6 +169,7 @@ "source_definition", "source_journal_head", "source", + "source_evidence_binding", "skill_package", "artifact_revision", "artifact_handle", @@ -173,6 +179,7 @@ "artifact_head", "memory_entry_version", "memory_entry_head", + "atomic_memory_evidence_history", "candidate_version", "candidate_head", "topic_memory_publication", @@ -299,6 +306,11 @@ class _RecordSpec: ("scope_id", "source_type", "source_id"), ("payload", "journal_position"), ), + "source_evidence_binding": _RecordSpec( + SOURCE_EVIDENCE_BINDINGS_TABLE, + ("scope_id", "source_type", "source_id"), + ("material_digest", "definition_name", "definition_version", "manifest_fingerprint", "declaration"), + ), "skill_package": _RecordSpec( SKILL_PACKAGES_TABLE, ("scope_id", "tree_digest"), @@ -373,6 +385,11 @@ class _RecordSpec: ("scope_id", "memory_artifact_id", "entry_id"), ("family", "head_revision", "entry_version_id", "entry_content_hash"), ), + "atomic_memory_evidence_history": _RecordSpec( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + ("scope_id", "artifact_id", "revision"), + ("sources",), + ), "candidate_version": _RecordSpec( CANDIDATE_VERSIONS_TABLE, ("scope_id", "candidate_id", "version"), @@ -961,14 +978,9 @@ async def _export_rows( return # Definition manifests are global, but export authority is scope-bound. - # Include only immutable declarations referenced by selected observations. - definitions: set[tuple[str, str]] = set() - async for row in _export_rows(connection, "source", scopes): - observation = _source_observation(_record_from_row("source", row)) - if observation is not None: - definitions.add((observation.source_type, observation.definition_version)) + # Include immutable manifests referenced by observations or retained bindings. table = SOURCE_DEFINITION_MANIFESTS_TABLE - for name, definition_version in sorted(definitions): + for name, definition_version in sorted(await _referenced_definitions(connection, scopes)): row = ( ( await connection.execute( @@ -981,10 +993,31 @@ async def _export_rows( .one_or_none() ) if row is None: - raise BundleFormatError("observation references missing Source definition") + raise BundleFormatError("evidence or observation references missing Source definition") yield cast(Mapping[str, Any], row) +async def _referenced_definitions(connection: AsyncConnection, scopes: tuple[str, ...]) -> set[tuple[str, str]]: + """Collect Definition identities whose immutable manifest must travel with the scope.""" + + definitions: set[tuple[str, str]] = set() + async for row in _export_rows(connection, "source", scopes): + observation = _source_observation(_record_from_row("source", row)) + if observation is not None: + definitions.add((observation.source_type, observation.definition_version)) + async for binding in _export_rows(connection, "source_evidence_binding", scopes): + if binding["manifest_fingerprint"] is not None: + definitions.add((str(binding["definition_name"]), str(binding["definition_version"]))) + async for row in _export_rows(connection, "atomic_memory_evidence_history", scopes): + record = _record_from_row("atomic_memory_evidence_history", row) + definitions.update( + (source.definition_name, source.definition_version) + for source in _evidence_history_value(record) + if source.manifest_fingerprint is not None + ) + return definitions + + def _source_observation(record: _Record) -> SourceObservation | None: try: value = json.loads(cast(bytes, _database_value(record.payload["payload"]))) @@ -1429,6 +1462,30 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: strings("fingerprint") bytes_values("manifest") _source_definition(record) + elif kind == "source_evidence_binding": + strings("material_digest", "definition_name", "definition_version") + strings("manifest_fingerprint", nullable=("manifest_fingerprint",)) + bytes_values("declaration") + try: + MemoryEvidenceDeclaration.model_validate_json(cast(bytes, _database_value(payload["declaration"]))) + except (ValidationError, ValueError, TypeError) as error: + raise BundleFormatError("Source evidence binding has an invalid declaration") from error + if ( + payload["definition_name"] != identity["source_type"] + or not _valid_digest(payload["material_digest"]) + or (payload["manifest_fingerprint"] is not None and not _valid_digest(payload["manifest_fingerprint"])) + ): + raise BundleFormatError("Source evidence binding identity or digest is invalid") + elif kind == "atomic_memory_evidence_history": + bytes_values("sources") + sources = _evidence_history_value(record) + if any( + not _valid_digest(source.material_digest) + or source.definition_name != source.source_ref.source_type + or (source.manifest_fingerprint is not None and not _valid_digest(source.manifest_fingerprint)) + for source in sources + ): + raise BundleFormatError("Atomic Memory evidence has invalid bindings") elif kind == "source": bytes_values("payload") integers("journal_position", positive=True) @@ -1640,6 +1697,19 @@ def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObserva return value +def _evidence_history_value(record: _Record, /) -> tuple[SourceEvidence, ...]: + try: + value = json.loads(cast(bytes, _database_value(record.payload["sources"]))) + except (ValueError, UnicodeDecodeError, TypeError) as error: + raise BundleFormatError("Atomic Memory evidence payload is not valid JSON") from error + if not isinstance(value, list): + raise BundleFormatError("Atomic Memory evidence payload is not a list") + try: + return tuple(SourceEvidence.model_validate(item) for item in value) + except ValidationError as error: + raise BundleFormatError("Atomic Memory evidence payload has an invalid Source entry") from error + + def _validate_datetime_text(value: object, /) -> None: try: parsed = datetime.fromisoformat(cast(str, value)) @@ -1972,6 +2042,46 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis }, ): raise BundleFormatError("observation references missing or mismatched Source definition") + elif record.record_type == "source_evidence_binding": + if not contains( + "source", + {"scope_id": scope, "source_type": identity["source_type"], "source_id": identity["source_id"]}, + ): + raise BundleFormatError("Source evidence binding references missing Source") + fingerprint = payload["manifest_fingerprint"] + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": payload["definition_name"], + "definition_version": payload["definition_version"], + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Source evidence binding references missing Source definition") + elif record.record_type == "atomic_memory_evidence_history": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": "atomic-memory", + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing exact Artifact revision") + for source in _evidence_history_value(record): + if not contains("source", {"scope_id": scope, **source.source_ref.model_dump()}): + raise BundleFormatError("Atomic Memory evidence references missing Source") + fingerprint = source.manifest_fingerprint + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": source.definition_name, + "definition_version": source.definition_version, + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing Source definition") elif record.record_type in {"recurrence_match", "recurrence_observation"}: value = _recurrence_value(record) if not contains( diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index a4101aace3..a26f66eb65 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -34,7 +34,7 @@ from sqlalchemy import create_engine, event, insert, select, update from sqlalchemy.engine import make_url -from powercontext.artifacts import ArtifactAddress, ArtifactRef, StableHandle +from powercontext.artifacts import ArtifactAddress, ArtifactLineage, ArtifactRef, StableHandle from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent from powercontext.builtin.artifacts.atomic_memory.errors import InvalidAtomicMemoryStateError from powercontext.builtin.artifacts.experience import ExperienceContent @@ -83,6 +83,7 @@ from powercontext.limits import MAX_ARTIFACT_ID_LENGTH from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, SourceDefinitionRegistry, SourceRef, manifest_for_definition, @@ -93,6 +94,54 @@ from tests.e2e.atomic_memory_migration_fixture import _seed_and_migrate +def test_archive_preserves_accepted_native_evidence(tmp_path: Path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), memory_evidence=declaration),)) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "evidence.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(source_config, source_registry=registry) as source: + scope = await source.scopes.create( + ScopeDraft(title="Evidence", summary="Portable evidence", idempotency_key="evidence") + ) + context = await source.get(scope.scope_id) + native = await context.sources.resolve(NoteInput(note_id="native", body="Retained evidence.")) + await context.sources.add(native) + ref = SourceRef(source_type="note", source_id="native") + service = source.atomic_memory.service + async with source.database.transaction() as connection: + plan = await service.inspect_change( + connection, + scope.scope_id, + "rule", + AtomicMemoryContent(kind="fact", text="Preserve evidence."), + None, + lineage=ArtifactLineage(sources=(ref,)), + ) + prepared = await service.prepare_change(plan) + async with source.database.transaction() as connection: + await service.commit(connection, prepared, None) + expected = await source.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert expected.sources[0].declaration == declaration + await source.portability.export([scope.scope_id], archive, authorize=authorize) + inspection = await source.portability.inspect(archive) + assert inspection.records_by_type["source_evidence_binding"] == 1 + assert inspection.records_by_type["atomic_memory_evidence_history"] == 1 + assert inspection.records_by_type["source_definition"] == 1 + async with open_builtin_contexts(target_config, source_registry=registry) as target: + await target.portability.restore(archive) + async with open_builtin_contexts(target_config, source_registry=registry) as target: + actual = await target.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert actual == expected + + asyncio.run(scenario()) + + def test_topic_restore_can_retry_after_projection_failure_and_restart(tmp_path: Path) -> None: async def authorize(_scopes: tuple[str, ...]) -> None: pass From d215c98ada6ac3ef7fc0c4e5d5045be7e104605c Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 28 Oct 2026 13:27:19 +0800 Subject: [PATCH 22/26] feat(memory): add rebuildable source evidence projection Persist exact-revision Atomic Memory evidence as an index-owned projection beside immutable retained history. Transactional body writes publish the accepted declaration and material snapshot; portable restore, legacy Memory conversion, and maintenance rebuild derive neutral facts from retained inputs only, never from today's registry. Extend the Source Definition boundary with a versioned MemoryEvidenceDeclaration (authority, verification, declaration_version), supplied only by trusted adapters or an exact remote manifest identity. Missing, invalid, or unverifiable declarations stay untrusted/unknown. Search order, public scores, bodies, hashes, citations, and write authority are unchanged. Register additive v1_5_0_002 migrations for SQLite and OceanBase without altering earlier revision fingerprints. Document the adapter boundary and projection lifecycle in the bilingual Sources workflow and RFC 1652. Verified: 7624 passed, 754 skipped; real OceanBase 4.4.2.1 evidence conformance; focused recovery, migration, docs contract, and type checks. Refs oceanbase/powercontext#1975. --- docs/en/docs/workflows/sources.md | 27 ++++ .../rfcs/1652_memory_quality_and_lifecycle.md | 18 +++ docs/zh/docs/workflows/sources.md | 26 +++ .../rfcs/1652_memory_quality_and_lifecycle.md | 15 ++ release/upgrade-support.json | 4 +- .../persistence/atomic_memory_evidence.py | 43 ++++- .../persistence/atomic_memory_index.py | 32 +++- .../persistence/atomic_memory_index_schema.py | 22 ++- .../topic_memory_recovery/schema.json | 7 + .../migrations/atomic_memory_v1.py | 6 + .../migrations/atomic_projection.py | 24 ++- .../atomic_memory_evidence_projection.py | 61 +++++++ .../migrations/server_resources/manifest.json | 30 +++- .../mysql/versions/v1_5_0_002.py | 25 +++ .../schemas/mysql-v1_5_0_002.json | 102 ++++++++++++ .../schemas/sqlite-v1_5_0_002.json | 5 + ...0_002_atomic_memory_evidence_projection.py | 31 ++++ .../server_sources/1.4.0/manifest.json | 30 +++- .../persistence/migrations/vector_profile.py | 5 + .../builtin/runtime/atomic_memory_rebuild.py | 151 +++++++++++++++--- .../builtin/runtime/relational.py | 5 + .../test_atomic_evidence_upgrade_history.py | 83 ++++++++-- .../test_full_server_migrations.py | 4 +- .../test_mysql_migration_runner.py | 16 +- .../test_owner_recovery_migration.py | 52 ++++-- .../test_portable_storage_migration.py | 4 +- .../test_release140_server_migrations.py | 2 +- .../e2e/test_atomic_memory_source_evidence.py | 31 +++- tests/e2e/test_managed_archive_recovery.py | 6 +- tests/e2e/test_observability.py | 13 +- tests/e2e/test_portable_archive.py | 7 + .../native/test_oceanbase_engine_recovery.py | 2 +- .../native/test_personal_service_lifecycle.py | 2 +- tests/test_managed_atomic_maintenance.py | 4 +- tests/test_managed_processing_maintenance.py | 2 +- tests/test_migration_revision_names.py | 6 +- .../test_registered_source_maintenance_cli.py | 2 +- 37 files changed, 816 insertions(+), 89 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence_projection.py diff --git a/docs/en/docs/workflows/sources.md b/docs/en/docs/workflows/sources.md index 78de15d770..6874f1fb0d 100644 --- a/docs/en/docs/workflows/sources.md +++ b/docs/en/docs/workflows/sources.md @@ -33,4 +33,31 @@ content under the same identity as if it were a new observation. - Subject evidence for a Scope profile: follow [Scope profiles](use-profiles.md). - Application or evaluation events: use the application's adapter; [Bub capture](../integrations/evaluation.md) is opt-in. +## Declare source authority + +A trusted adapter can declare how its material should be trusted. The optional `memory_evidence` field on a Source +Definition carries a versioned `MemoryEvidenceDeclaration` with `authority` (`untrusted`, `user_asserted`, +`repository_attested`, or `system_attested`) and `verification` (`unknown`, `verified`, or `not_verified`). Only +adapter or deployment code supplies it; Source text, references, model output, and retrieval rank cannot. + +```python +from powercontext.sources import AdapterSourceDefinition, MemoryEvidenceDeclaration + +definition = AdapterSourceDefinition( + existing_adapter, + memory_evidence=MemoryEvidenceDeclaration( + authority="repository_attested", + verification="not_verified", + ), +) +``` + +Remote Source observations configure the same declaration in trusted deployment code, keyed by the exact Definition +name, version, and manifest fingerprint. A missing, invalid, or unavailable declaration resolves to +`untrusted`/`unknown`; it does not block capture, processing, or existing retrieval. + +Each Memory revision records the declaration and exact Source materialization accepted with it. That retained evidence +is authoritative and versioned. The index projection beside it is derived: maintenance can discard and rebuild it +without consulting today's registry. Declarations add no ranking weight, score, or access right. + Source writes and reads are subject to the selected [Scope and access policy](scopes-and-access.md). diff --git a/docs/en/rfcs/1652_memory_quality_and_lifecycle.md b/docs/en/rfcs/1652_memory_quality_and_lifecycle.md index c6e21f4d2f..0deff631b1 100644 --- a/docs/en/rfcs/1652_memory_quality_and_lifecycle.md +++ b/docs/en/rfcs/1652_memory_quality_and_lifecycle.md @@ -222,6 +222,24 @@ candidate pool or delivered Context Pack occupied by a lower-authority class, bu legitimate answer-bearing evidence. This is an opt-in experiment, not a stage-1 default or a substitute for the Source declaration contract. +## Current implementation amendment: source evidence declaration + +The Source evidence prerequisite is implemented as specified above. This section supersedes the "provisionally named" +wording and the statement that `SourceDefinition` carries no authority or verification declaration. + +- `SourceDefinition` and `AdapterSourceDefinition` expose an optional `memory_evidence` declaration with `authority`, + `verification`, and `declaration_version="1"`. +- Trusted adapter code, or deployment configuration keyed by an exact `(name, version, fingerprint)` Definition + manifest, supplies the declaration. Source text, `SourceRef`, model output, and retrieval rank cannot. +- Each Atomic Memory revision retains the accepted declaration with its exact Source materialization and manifest + fingerprint. A separate index-owned projection holds the same facts for that revision and can be discarded and + rebuilt from retained evidence without consulting the current registry. +- Missing, invalid, or unverifiable declarations remain `untrusted`/`unknown` for reads and rebuilds. There is no + trust backfill, and existing write authority, generation eligibility, body text, content hashes, citations, and + search contracts are unchanged. +- Stage-1 quality ranking, lifecycle actions, and receipt-linked usage feedback remain unimplemented. This slice is + their prerequisite, not a substitute for them. + ## Independent quality dimensions The policy keeps these dimensions separate: diff --git a/docs/zh/docs/workflows/sources.md b/docs/zh/docs/workflows/sources.md index 98f7b7519a..380e9f8cd7 100644 --- a/docs/zh/docs/workflows/sources.md +++ b/docs/zh/docs/workflows/sources.md @@ -31,4 +31,30 @@ Work Contract 和 Task Outcome 也会保留 Source 证据。 - Scope Profile 的主体证据：按 [Scope Profile](use-profiles.md)写入。 - 应用或评测事件：使用应用对应适配器；[Bub 采集](../integrations/evaluation.md)需要显式启用。 +## 声明 Source 权威性 + +受信任的 adapter 可以声明其材料的可信程度。Source Definition 上的可选 `memory_evidence` 字段携带版本化的 +`MemoryEvidenceDeclaration`，其中 `authority` 取 `untrusted`、`user_asserted`、`repository_attested` 或 +`system_attested`，`verification` 取 `unknown`、`verified` 或 `not_verified`。它只能由 adapter 或部署代码提供， +Source 正文、引用、模型输出和检索排名都不能。 + +```python +from powercontext.sources import AdapterSourceDefinition, MemoryEvidenceDeclaration + +definition = AdapterSourceDefinition( + existing_adapter, + memory_evidence=MemoryEvidenceDeclaration( + authority="repository_attested", + verification="not_verified", + ), +) +``` + +远程 Source observation 在受信任的部署代码中配置同一声明，键为精确的 Definition 名称、版本与 manifest +fingerprint。声明缺失、无效或不可用时解析为 `untrusted`/`unknown`，不阻塞采集、处理或既有检索。 + +每个 Memory revision 记录当时接受的声明与精确 Source materialization。保留的证据是权威且带版本的；其旁的索引 +投影是派生数据，维护可以丢弃它并从保留事实重建，且不查询当前 registry。声明不增加排序权重、不改变分数， +也不授予访问权限。 + Source 读写受所选 [Scope 与访问策略](scopes-and-access.md)约束。 diff --git a/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md b/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md index 06dce9a62e..3e1d5779e1 100644 --- a/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md +++ b/docs/zh/rfcs/1652_memory_quality_and_lifecycle.md @@ -195,6 +195,21 @@ Memory entry 写入时，必须把声明版本及解析后的 authority/verifica 进行对比：限制低 authority 类别在固定候选池或最终 Context Pack 中占用的位置/字节份额，但必须保留显式路径，允许 确实能回答问题的证据进入。它是 opt-in 的评测实验，不是第一阶段默认行为，也不能替代 Source 声明 contract。 +## 当前实现修订：Source 证据声明 + +上文要求的 Source 证据前置条件已按此规格实现。本节取代“暂称”的表述，以及 `SourceDefinition` 不携带 +authority/verification 声明的说法。 + +- `SourceDefinition` 与 `AdapterSourceDefinition` 提供可选的 `memory_evidence` 声明，包含 `authority`、 + `verification` 与 `declaration_version="1"`。 +- 声明只能由受信任的 adapter 代码，或在部署配置中以精确 `(name, version, fingerprint)` Definition manifest + 为键提供。Source 正文、`SourceRef`、模型输出与检索排名都不能。 +- 每个 Atomic Memory revision 保留当时接受的声明、精确 Source materialization 与 manifest fingerprint。同一 + revision 的相同事实另有索引拥有的派生投影；该投影可丢弃，并由保留证据重建，不查询当前 registry。 +- 声明缺失、无效或不可验证时，读取与重建都保持 `untrusted`/`unknown`。不做信任回填，既有写入权威、生成资格、 + 正文、content hash、citation 与搜索契约不变。 +- 第一阶段质量排序、生命周期操作与基于投递回执的使用反馈仍未实现。该能力只是其前置条件，不替代它们。 + ## 独立质量维度 四个维度必须独立： diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 69f3fd80f0..de5a4cd808 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "d200c27f8864723a6430fb47a8ef005f405d2ed92f2b48f35707d657ab42f75a", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB full-server-code and full-server-vector-code are temporarily manual-only due to native DDL waits with the available engine. Main and publication require the other ten; no manual result is needed for publication. The two deferred configurations remain unverified and are excluded from generated supported upgrade declarations. Reports retain their identities and explicitly record incomplete full-matrix acceptance. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the same automatic matrix, manual-only configurations and publication restrictions. Revision v1_5_0_001 follows v1_4_1_004 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged." + "persistence_sha256": "5e2817a293f90457217ce3d6004aeecec5a022fe0f3eef402b53006003f9acd9", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB full-server-code and full-server-vector-code are temporarily manual-only due to native DDL waits with the available engine. Main and publication require the other ten; no manual result is needed for publication. The two deferred configurations remain unverified and are excluded from generated supported upgrade declarations. Reports retain their identities and explicitly record incomplete full-matrix acceptance. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the same automatic matrix, manual-only configurations and publication restrictions. Revision v1_5_0_001 follows v1_4_1_004 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_002 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py index 99d5c38311..7170a0ce83 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -24,11 +24,14 @@ from aiosqlite import Connection as SQLiteConnection from pydantic import BaseModel, ConfigDict, ValidationError from sqlalchemy import insert, select +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE from powercontext.builtin.persistence.sources import SourceRepository, _require_identity from powercontext.limits import MAX_SCOPE_ID_LENGTH from powercontext.sources import SourceRef @@ -49,8 +52,8 @@ class AtomicMemoryEvidenceAnnotation(BaseModel): class AtomicMemoryEvidenceRepository: """Join caller transactions; retained inputs are never deleted with the index. - The annotation is derived from retained inputs at read time. There is no - separate cache to confuse with accepted history or join into search. + Accepted history is immutable. The index-owned projection can be discarded + and replaced from verified history without joining it into search. """ async def save( @@ -105,6 +108,42 @@ async def read( payload = await self._payload(connection, scope_id, artifact_ref) return None if payload is None else self._decode(scope_id, artifact_ref, payload) + async def read_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + payload = await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + async def replace_projection(self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation) -> None: + reference = self._reference(annotation.scope_id, annotation.artifact_ref) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + payload = rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]) + if connection.dialect.name == "sqlite": + statement = sqlite_insert(table) + statement = statement.on_conflict_do_update( + index_elements=("scope_id", "artifact_id", "revision"), set_={"sources": statement.excluded.sources} + ) + else: + statement = mysql_insert(table) + statement = statement.on_duplicate_key_update(sources=statement.inserted.sources) + await connection.execute( + statement, + { + "scope_id": annotation.scope_id, + "artifact_id": reference.artifact_id, + "revision": reference.revision, + "sources": payload, + }, + ) + async def verify( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 8ce323ea19..192f4bbe75 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -48,7 +48,10 @@ AtomicMemoryEvidenceRepository, ) from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES -from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + ATOMIC_MEMORY_PROJECTION_FORMAT, +) from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter @@ -209,6 +212,14 @@ async def read_evidence( self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def project_evidence( + self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation, / + ) -> None: ... + + async def read_evidence_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -299,7 +310,8 @@ async def capture_evidence(self, connection: AsyncConnection, scope_id: str, rec and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None ): evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) - await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + annotation = await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + await self.index.project_evidence(connection, annotation) async def publish( self, @@ -594,7 +606,11 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table, *ATOMIC_MEMORY_EVIDENCE_TABLES) + self.tables: tuple[Table, ...] = ( + table, + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + *ATOMIC_MEMORY_EVIDENCE_TABLES, + ) self._evidence = AtomicMemoryEvidenceRepository() self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( @@ -616,6 +632,16 @@ async def read_evidence( ) -> AtomicMemoryEvidenceAnnotation | None: return await self._evidence.read(connection, scope_id, artifact_ref) + async def project_evidence( + self, connection: AsyncConnection, annotation: AtomicMemoryEvidenceAnnotation, / + ) -> None: + await self._evidence.replace_projection(connection, annotation) + + async def read_evidence_projection( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read_projection(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /) -> None: """Audit vector deployment readiness at startup, outside ordinary bounded search.""" profile = self.capabilities.embedding_profile diff --git a/src/powercontext/builtin/persistence/atomic_memory_index_schema.py b/src/powercontext/builtin/persistence/atomic_memory_index_schema.py index 9210fba152..6c30011b02 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index_schema.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index_schema.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The single rebuildable current Atomic Memory projection.""" +"""Rebuildable current and exact-version Atomic Memory index projections.""" from typing import Any from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, MetaData, Table, Text -from sqlalchemy.dialects.mysql import MEDIUMTEXT +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT from sqlalchemy.types import TypeEngine from powercontext.builtin.persistence.tables import identity_string @@ -26,6 +26,17 @@ ATOMIC_MEMORY_CURRENT_TABLE_NAME = "pc_atomic_memory_current" ATOMIC_MEMORY_PROJECTION_FORMAT = "powercontext.atomic-memory.current.v1" +# This table is disposable index state, never retained archive authority. +ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE = Table( + "pc_atomic_memory_evidence", + MetaData(), + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_revision"), +) + def atomic_memory_current_table(embedding_type: TypeEngine[Any] | None = None, /) -> Table: """Build one backend-owned table, with its deployment's native vector type. @@ -61,4 +72,9 @@ def atomic_memory_current_table(embedding_type: TypeEngine[Any] | None = None, / ) -__all__ = ["ATOMIC_MEMORY_CURRENT_TABLE_NAME", "ATOMIC_MEMORY_PROJECTION_FORMAT", "atomic_memory_current_table"] +__all__ = [ + "ATOMIC_MEMORY_CURRENT_TABLE_NAME", + "ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE", + "ATOMIC_MEMORY_PROJECTION_FORMAT", + "atomic_memory_current_table", +] diff --git a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json index f7159e2595..d436535d18 100644 --- a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json +++ b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json @@ -101,6 +101,12 @@ "pc_atomic_memory_current_fts_idx", "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" ], + [ + "table", + "pc_atomic_memory_evidence", + "pc_atomic_memory_evidence", + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ], [ "table", "pc_atomic_memory_evidence_history", @@ -147,6 +153,7 @@ "empty_tables": [ "pc_atomic_memory_current", "pc_atomic_memory_current_fts", + "pc_atomic_memory_evidence", "pc_memory_artifact_archive" ] } diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py index 7dffbfe80d..aea58eb4c8 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_v1.py @@ -1560,6 +1560,12 @@ async def apply_atomic_memory_migration( # noqa: C901 - One ordered offline mai raise AtomicMemoryMigrationError((str(error),)) from error async with database.transaction() as connection: await drop_legacy_citation_columns(connection) + if "pc_artifacts" in await _tables(connection): + from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_evidence + + # Imported revisions never accepted a declaration; derive neutral facts + # from the converted lineage without manufacturing historical snapshots. + await rebuild_atomic_memory_evidence(connection, index) async with database.transaction() as connection: residual = await residual_issues(connection, await _tables(connection), thorough=False) if rebuilt_projection: diff --git a/src/powercontext/builtin/persistence/migrations/atomic_projection.py b/src/powercontext/builtin/persistence/migrations/atomic_projection.py index c700267b12..492b8d8240 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_projection.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_projection.py @@ -23,6 +23,11 @@ from sqlalchemy import Connection, select, tuple_ from typing_extensions import override +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex @@ -63,7 +68,11 @@ def _sqlite_helper_digest(connection, batch_size): class AtomicProjectionMigrationRunner(ManagedMaintenanceRunner): operation = "atomic_projection" - affected_objects: tuple[str, ...] = ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + affected_objects: tuple[str, ...] = ( + "pc_atomic_memory_current", + "pc_atomic_memory_current_fts", + "pc_atomic_memory_evidence", + ) def __init__(self, *args, **kwargs): super().__init__( @@ -75,9 +84,9 @@ def __init__(self, *args, **kwargs): **kwargs, ) self.affected_objects = ( - ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + ("pc_atomic_memory_current", "pc_atomic_memory_current_fts", "pc_atomic_memory_evidence") if isinstance(self.connections.config, SQLiteConfig) - else ("pc_atomic_memory_current",) + else ("pc_atomic_memory_current", "pc_atomic_memory_evidence") ) def _index(self, connection): @@ -90,7 +99,11 @@ def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str try: report = finish( atomic_memory_rebuild.verify_atomic_memory_current( - pinned(connection), index, batch_size=self.batch_size, require_evidence=True + pinned(connection), + index, + batch_size=self.batch_size, + require_evidence=True, + require_evidence_projection=True, ) ) except Exception as error: @@ -122,6 +135,9 @@ def _rows_digest(self, connection, current): ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACT_LINEAGE_ARTIFACTS_TABLE, current, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, ): snapshot.update(table.name.encode()) keys = tuple(table.primary_key.columns) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py new file mode 100644 index 0000000000..e6b90989cb --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rebuild disposable exact-revision evidence from retained declarations.""" + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import require_atomic_memory_evidence_schema +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.pinned import finish, pinned +from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex +from powercontext.builtin.runtime.atomic_memory_rebuild import ( + rebuild_atomic_memory_evidence, + verify_atomic_memory_evidence, +) + + +def _index(connection, configuration): + profile = ( + EmbeddingProfile.model_validate(configuration["embedding_profile"]) + if configuration.get("embedding_enabled") + else None + ) + return ( + SQLiteAtomicMemoryIndex(profile) if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex(profile) + ) + + +def preflight(connection, *, configuration): + # Earlier pending revisions may not yet have created retained evidence tables. + from sqlalchemy import inspect + + if "pc_atomic_memory_evidence_schema" in inspect(connection).get_table_names(): + finish(require_atomic_memory_evidence_schema(pinned(connection))) + + +def apply(connection, *, configuration): + finish(rebuild_atomic_memory_evidence(pinned(connection), _index(connection, configuration))) + validate(connection, configuration=configuration) + + +def validate(connection, *, configuration): + report = finish(verify_atomic_memory_evidence(pinned(connection), _index(connection, configuration))) + if not report.ready: + raise MigrationError( + "projection_verification_failed", "Atomic Memory evidence projections differ from retained declarations." + ) + + +verify = validate diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index b9949ec3ec..4ed48b055c 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -28,6 +28,10 @@ "v1_5_0_001": [ "schemas/sqlite-v1_5_0_001.json", "schemas/mysql-v1_5_0_001.json" + ], + "v1_5_0_002": [ + "schemas/sqlite-v1_5_0_002.json", + "schemas/mysql-v1_5_0_002.json" ] }, "sqlite_baselines": { @@ -51,6 +55,9 @@ ], "v1_5_0_001": [ "4817f09fe11f442923b2ae59040e71691fc171a6c15102b9db4df496e78f0fc4" + ], + "v1_5_0_002": [ + "36bec00492ba0c8532d3178f1d94f0281e9d5dcaf1fa64598e529b43042a2d56" ] }, "affected_objects": [ @@ -148,6 +155,7 @@ "pc_access_owner_recovery", "pc_atomic_memory_evidence_schema", "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", "pc_source_evidence_bindings" ], "readiness_scope": "server", @@ -203,6 +211,20 @@ "seekdb", "oceanbase" ] + }, + { + "id": "atomic_memory_evidence_projection", + "revision": "v1_5_0_002", + "kind": "projection", + "module": "atomic_memory_evidence_projection.py", + "depends_on": [ + "atomic_memory_evidence" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -224,6 +246,11 @@ "server_upgrade", "portable_storage", "atomic_memory_evidence" + ], + "v1_5_0_002": [ + "server_upgrade", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -240,7 +267,8 @@ "schemas/mysql-s1401.json", "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", - "schemas/mysql-v1_5_0_001.json" + "schemas/mysql-v1_5_0_001.json", + "schemas/mysql-v1_5_0_002.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py new file mode 100644 index 0000000000..b91d6985a3 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/v1_5_0_002.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Register the index-owned evidence projection through the shared DDL executor.""" + +from alembic import context + +revision = "v1_5_0_002" +down_revision = "v1_5_0_001" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json new file mode 100644 index 0000000000..8ec422cb11 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-v1_5_0_002.json @@ -0,0 +1,102 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "v1_5_0_002": { + "statements": [ + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ], + "changes": [ + { + "pc_atomic_memory_evidence": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "sources": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_atomic_memory_evidence_revision": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + } + } + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json new file mode 100644 index 0000000000..9a726e95fa --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-v1_5_0_002.json @@ -0,0 +1,5 @@ +{ + "create": [ + "CREATE TABLE pc_atomic_memory_evidence (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_revision CHECK (revision > 0)\n)" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence_projection.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence_projection.py new file mode 100644 index 0000000000..f727ee5d3e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/v1_5_0_002_atomic_memory_evidence_projection.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Register disposable exact-revision Atomic Memory evidence projection.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "v1_5_0_002" +down_revision = "v1_5_0_001" + + +def upgrade(): + statements = json.loads((Path(__file__).parents[1] / "schemas/sqlite-v1_5_0_002.json").read_text()) + connection = op.get_bind() + for statement in statements["create"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json index 2306773b83..53644d7deb 100644 --- a/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_sources/1.4.0/manifest.json @@ -27,6 +27,10 @@ "v1_5_0_001": [ "schemas/sqlite-v1_5_0_001.json", "schemas/mysql-v1_5_0_001.json" + ], + "v1_5_0_002": [ + "schemas/sqlite-v1_5_0_002.json", + "schemas/mysql-v1_5_0_002.json" ] }, "sqlite_baselines": { @@ -50,6 +54,9 @@ ], "v1_5_0_001": [ "4817f09fe11f442923b2ae59040e71691fc171a6c15102b9db4df496e78f0fc4" + ], + "v1_5_0_002": [ + "36bec00492ba0c8532d3178f1d94f0281e9d5dcaf1fa64598e529b43042a2d56" ] }, "affected_objects": [ @@ -147,6 +154,7 @@ "pc_access_owner_recovery", "pc_atomic_memory_evidence_schema", "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", "pc_source_evidence_bindings" ], "readiness_scope": "server", @@ -207,6 +215,20 @@ "seekdb", "oceanbase" ] + }, + { + "id": "atomic_memory_evidence_projection", + "revision": "v1_5_0_002", + "kind": "projection", + "module": "atomic_memory_evidence_projection.py", + "depends_on": [ + "atomic_memory_evidence" + ], + "backends": [ + "sqlite", + "seekdb", + "oceanbase" + ] } ], "domain_readiness": { @@ -228,6 +250,11 @@ "server_upgrade", "portable_storage", "atomic_memory_evidence" + ], + "v1_5_0_002": [ + "server_upgrade", + "portable_storage", + "atomic_memory_evidence_projection" ] }, "retained_fingerprints": { @@ -244,7 +271,8 @@ "schemas/mysql-s1401.json", "schemas/mysql-s1402.json", "schemas/mysql-v1_4_1_004.json", - "schemas/mysql-v1_5_0_001.json" + "schemas/mysql-v1_5_0_001.json", + "schemas/mysql-v1_5_0_002.json" ], "mysql_history": "mysql", "mysql_schema_extensions": { diff --git a/src/powercontext/builtin/persistence/migrations/vector_profile.py b/src/powercontext/builtin/persistence/migrations/vector_profile.py index 6f81d6f5eb..180c882e8e 100644 --- a/src/powercontext/builtin/persistence/migrations/vector_profile.py +++ b/src/powercontext/builtin/persistence/migrations/vector_profile.py @@ -90,6 +90,11 @@ def sqlite_layouts(self, directory: Path) -> dict[str, list[list[Any]]]: layouts["v1_5_0_001"] = layouts["v1_4_1_004"] + [ ["table", name, name, statement] for statement in evidence if (name := _created_table(statement)) ] + if "v1_5_0_002" in manifest["revision_resources"]: + evidence = json.loads((directory / "schemas/sqlite-v1_5_0_002.json").read_text(encoding="utf-8"))["create"] + layouts["v1_5_0_002"] = layouts["v1_5_0_001"] + [ + ["table", name, name, statement] for statement in evidence if (name := _created_table(statement)) + ] for entries in layouts.values(): entries.sort(key=lambda row: (row[0], row[1])) return layouts diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 5ead8c6bb8..1b2ba69d4c 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -17,7 +17,7 @@ from __future__ import annotations import json -from collections.abc import Callable +from collections.abc import AsyncIterator, Callable from hashlib import sha256 from time import perf_counter @@ -33,7 +33,10 @@ from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository -from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -44,7 +47,10 @@ drop_obsolete_atomic_memory_projection, load_atomic_memory_tags, ) -from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + ATOMIC_MEMORY_PROJECTION_FORMAT, +) from powercontext.builtin.persistence.atomic_memory_readiness import ( atomic_memory_readiness_issues, ensure_supported_atomic_memory_schema, @@ -119,17 +125,23 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) -async def _verify_history( - connection: AsyncConnection, artifacts: ArtifactRepository, batch_size: int, sources: SourceRepository | None -) -> dict[str, int]: +async def _evidence_history( + connection: AsyncConnection, + artifacts: ArtifactRepository, + batch_size: int, + sources: SourceRepository | None, + *, + selected_scope: str | None = None, +) -> AsyncIterator[AtomicMemoryEvidenceAnnotation]: table = ARTIFACTS_TABLE evidence = AtomicMemoryEvidenceRepository() after: tuple[str, str, int] | None = None - revisions = neutral_sources = 0 while True: statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( table.c.family == AtomicMemory.family ) + if selected_scope is not None: + statement = statement.where(table.c.scope_id == selected_scope) if after is not None: statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) rows = ( @@ -143,16 +155,75 @@ async def _verify_history( scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) artifact = await artifacts.get(connection, scope_id, reference) - annotation = await evidence.verify( - connection, scope_id, reference, artifact.lineage.sources, sources=sources - ) - revisions += 1 - neutral_sources += sum( - source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" - for source in annotation.sources - ) + yield await evidence.verify(connection, scope_id, reference, artifact.lineage.sources, sources=sources) after = scope_id, artifact_id, revision - return {"historical_revisions": revisions, "neutral_sources": neutral_sources} + + +def _obsolete_evidence(): + projection = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + return ~exists( + select(ARTIFACTS_TABLE.c.artifact_id).where( + ARTIFACTS_TABLE.c.scope_id == projection.c.scope_id, + ARTIFACTS_TABLE.c.family == AtomicMemory.family, + ARTIFACTS_TABLE.c.artifact_id == projection.c.artifact_id, + ARTIFACTS_TABLE.c.revision == projection.c.revision, + ) + ) + + +async def rebuild_atomic_memory_evidence( + connection: AsyncConnection, + index: AtomicMemoryIndex, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, + guard: Callable[[], None] = lambda: None, + scope_id: str | None = None, +) -> None: + """Rebuild disposable revision facts without changing history or invoking models. + + The maintenance caller owns schema admission, writer exclusion and the transaction. + """ + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") + artifacts = ArtifactRepository((AtomicMemory,)) + async for annotation in _evidence_history(connection, artifacts, batch_size, sources, selected_scope=scope_id): + guard() + await index.project_evidence(connection, annotation) + table = ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + statement = table.delete().where(_obsolete_evidence()) + if scope_id is not None: + statement = statement.where(table.c.scope_id == scope_id) + await connection.execute(statement) + + +async def _verify_history( + connection: AsyncConnection, + index: AtomicMemoryIndex, + artifacts: ArtifactRepository, + batch_size: int, + sources: SourceRepository | None, + require_projection: bool, +) -> dict[str, int]: + revisions = neutral_sources = differing = 0 + async for annotation in _evidence_history(connection, artifacts, batch_size, sources): + revisions += 1 + neutral_sources += sum( + source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" + for source in annotation.sources + ) + if require_projection: + projected = await index.read_evidence_projection(connection, annotation.scope_id, annotation.artifact_ref) + differing += projected != annotation + counts = {"historical_revisions": revisions, "neutral_sources": neutral_sources} + if require_projection: + counts["evidence_projection_differences"] = differing + int( + await connection.scalar( + select(func.count()).select_from(ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE).where(_obsolete_evidence()) + ) + or 0 + ) + return counts async def _record( @@ -221,6 +292,26 @@ async def _projection_differences( return differing +async def verify_atomic_memory_evidence( + connection: AsyncConnection, + index: AtomicMemoryIndex, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, +) -> AtomicMemoryProjectionRebuildReport: + """Validate exact revision facts independently of the deployed vector profile.""" + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") + await _require_authority(connection) + counts = await _verify_history(connection, index, ArtifactRepository((AtomicMemory,)), batch_size, sources, True) + differences = counts["evidence_projection_differences"] + return AtomicMemoryProjectionRebuildReport( + ready=not differences, + counts=counts, + errors=(f"{differences} evidence projections differ from retained history",) if differences else (), + ) + + async def verify_atomic_memory_current( connection: AsyncConnection, index: AtomicMemoryIndex, @@ -229,6 +320,7 @@ async def verify_atomic_memory_current( batch_size: int = 100, sources: SourceRepository | None = None, require_evidence: bool = False, + require_evidence_projection: bool = False, ) -> AtomicMemoryProjectionRebuildReport: """Check current projections and optionally each exact revision's retained evidence without business row locks. @@ -240,12 +332,20 @@ async def verify_atomic_memory_current( if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") - await _require_authority(connection, require_evidence=require_evidence) + await _require_authority(connection, require_evidence=require_evidence or require_evidence_projection) artifacts = ArtifactRepository((AtomicMemory,)) - history_counts = await _verify_history(connection, artifacts, batch_size, sources) if require_evidence else {} + history_counts = ( + await _verify_history(connection, index, artifacts, batch_size, sources, require_evidence_projection) + if require_evidence or require_evidence_projection + else {} + ) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) errors: list[str] = [] + if history_counts.get("evidence_projection_differences"): + errors.append( + f"{history_counts['evidence_projection_differences']} evidence projections differ from retained history" + ) after = None active = 0 while batch := await _identity_batch(connection, after, batch_size): @@ -325,7 +425,10 @@ async def rebuild_atomic_memory_current( await publisher.remove(connection, scope_id, artifact_id) after = scope_id, artifact_id await connection.execute(index.table.delete().where(_obsolete_current(index))) - return await verify_atomic_memory_current(connection, index, batch_size=batch_size, require_evidence=True) + await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, guard=guard) + return await verify_atomic_memory_current( + connection, index, batch_size=batch_size, require_evidence=True, require_evidence_projection=True + ) async def rebuild_atomic_memory_projection( @@ -404,8 +507,14 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete + await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, sources=sources) report = await verify_atomic_memory_current( - connection, index, batch_size=batch_size, sources=sources, require_evidence=True + connection, + index, + batch_size=batch_size, + sources=sources, + require_evidence=True, + require_evidence_projection=True, ) return report.model_copy( update={ @@ -423,6 +532,8 @@ async def rebuild_atomic_memory_projection( __all__ = [ "AtomicMemoryProjectionRebuildReport", "rebuild_atomic_memory_current", + "rebuild_atomic_memory_evidence", "rebuild_atomic_memory_projection", "verify_atomic_memory_current", + "verify_atomic_memory_evidence", ] diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 03aebe12aa..52fb49ddbd 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -1422,6 +1422,8 @@ async def handoff_scope_ids(self) -> tuple[str, ...]: async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> None: """Rebuild target-local search projections after a logical restore.""" + from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_evidence + for scope_id in scope_ids: async with self.database.transaction() as connection: await self.atomic_memory_index.initialize(connection) @@ -1467,6 +1469,9 @@ async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> N ) if projected != rebuilt: raise RuntimeError("restored Atomic Memory projection is incomplete") # noqa: TRY003 + await rebuild_atomic_memory_evidence( + connection, self.atomic_memory_index, sources=self.repositories.sources, scope_id=scope_id + ) topic_heads = tuple( ( await connection.execute( diff --git a/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py index 8a327acf29..0d78ac0787 100644 --- a/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py +++ b/tests/builtin/persistence/test_atomic_evidence_upgrade_history.py @@ -33,9 +33,11 @@ from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest EVIDENCE_REVISION = "v1_5_0_001" +PROJECTION_REVISION = "v1_5_0_002" EVIDENCE_TABLES = { "pc_atomic_memory_evidence_schema", "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", "pc_source_evidence_bindings", } @@ -71,25 +73,29 @@ def previous_upstream_bundle(directory, embedding): shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - removed_resources = set(manifest["revision_resources"].pop(EVIDENCE_REVISION)) - removed_steps = [step for step in manifest["domain_steps"] if step["revision"] == EVIDENCE_REVISION] - manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != EVIDENCE_REVISION] + appended = {EVIDENCE_REVISION, PROJECTION_REVISION} + removed_resources = {resource for revision in appended for resource in manifest["revision_resources"].pop(revision)} + removed_steps = [step for step in manifest["domain_steps"] if step["revision"] in appended] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in appended] for step in removed_steps: removed_resources.update((step["module"], *step.get("resources", ()))) retained_resources = {resource for resources in manifest["revision_resources"].values() for resource in resources} for step in manifest["domain_steps"]: retained_resources.update((step["module"], *step.get("resources", ()))) - revision = current.script.get_revision(EVIDENCE_REVISION) - removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) - removed_resources.add("mysql/versions/v1_5_0_001.py") + for name in appended: + revision = current.script.get_revision(name) + removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed_resources.add(f"mysql/versions/{name}.py") for resource in removed_resources - retained_resources: (directory / resource).unlink() - del manifest["sqlite_fingerprints"][EVIDENCE_REVISION] - del manifest["domain_readiness"][EVIDENCE_REVISION] + for revision in appended: + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] manifest["mysql_state_layouts"] = [ resource for resource in manifest["mysql_state_layouts"] if resource not in removed_resources ] - manifest.get("mysql_wrappers", {}).pop(EVIDENCE_REVISION, None) + for revision in appended: + manifest.get("mysql_wrappers", {}).pop(revision, None) manifest["affected_objects"] = [name for name in manifest["affected_objects"] if name not in EVIDENCE_TABLES] manifest_path.write_text(json.dumps(manifest), encoding="utf-8") return MigrationBundle(directory, embedding_profile=embedding) @@ -147,13 +153,13 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, plan = upgraded.plan() assert plan.state == "migration_required" assert plan.source_revision == "v1_4_1_004" - assert plan.target_revision == EVIDENCE_REVISION - assert plan.revisions == (EVIDENCE_REVISION,) - assert plan.domain_steps == ("atomic_memory_evidence",) + assert plan.target_revision == PROJECTION_REVISION + assert plan.revisions == (EVIDENCE_REVISION, PROJECTION_REVISION) + assert plan.domain_steps == ("atomic_memory_evidence", "atomic_memory_evidence_projection") assert upgraded.evidence_path.read_bytes() == old_evidence_bytes result = upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) assert result.changed - assert result.revision == EVIDENCE_REVISION + assert result.revision == PROJECTION_REVISION assert result.backup_state == "completed" # Reconstruct both bundle and runner: verification must use retained on-disk @@ -163,11 +169,11 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, completed_evidence = json.loads(completed_evidence_bytes) assert completed_evidence["state"] == "complete" assert {revision: completed_evidence["release_checksums"][revision] for revision in expected} == expected - assert set(completed_evidence["release_checksums"]) == {*expected, EVIDENCE_REVISION} + assert set(completed_evidence["release_checksums"]) == {*expected, EVIDENCE_REVISION, PROJECTION_REVISION} ready = reopened.plan() assert ready.state == "ready" assert ready.revisions == () - assert reopened.verify().revision == EVIDENCE_REVISION + assert reopened.verify().revision == PROJECTION_REVISION assert not apply(reopened).changed assert reopened.evidence_path.read_bytes() == completed_evidence_bytes with closing(sqlite3.connect(database)) as connection: @@ -181,6 +187,7 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, ("p0005",) ] assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence_history").fetchone() == (0,) + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence").fetchone() == (0,) assert connection.execute("SELECT COUNT(*) FROM pc_source_evidence_bindings").fetchone() == (0,) if embedding is not None: stored_profile = connection.execute("SELECT embedding_profile FROM pc_server_storage_profile").fetchone() @@ -196,3 +203,49 @@ def test_evidence_upgrade_accepts_upstream_history_and_preserves_data(tmp_path, assert reopened.evidence_path.read_bytes() == completed_evidence_bytes with closing(sqlite3.connect(database)) as connection: assert connection.execute("SELECT schema_version FROM pc_atomic_memory_evidence_schema").fetchall() == [] + + +@pytest.mark.parametrize("vectors", [False, True], ids=["lexical", "vectors"]) +def test_evidence_projection_upgrade_preserves_registered_evidence_revision(tmp_path, vectors): + embedding = EmbeddingProfile(profile_id="evidence-upgrade", model="fixture", dimension=3) if vectors else None + current = server_bundle(embedding_profile=embedding) + directory = tmp_path / "evidence-bundle" + shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + removed = manifest["revision_resources"].pop(PROJECTION_REVISION) + removed.append("atomic_memory_evidence_projection.py") + revision = current.script.get_revision(PROJECTION_REVISION) + removed.append(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed.append(f"mysql/versions/{PROJECTION_REVISION}.py") + for resource in removed: + (directory / resource).unlink() + del manifest["sqlite_fingerprints"][PROJECTION_REVISION] + del manifest["domain_readiness"][PROJECTION_REVISION] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] != PROJECTION_REVISION] + manifest["mysql_state_layouts"] = [name for name in manifest["mysql_state_layouts"] if name not in removed] + manifest["affected_objects"].remove("pc_atomic_memory_evidence") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + previous = MigrationBundle(directory, embedding_profile=embedding) + assert previous.head == EVIDENCE_REVISION + assert {revision: current.checksums[revision] for revision in previous.revisions} == previous.checksums + + database = tmp_path / "adjacent.db" + config = configuration(embedding) + old = SQLiteMigrationRunner(database, previous, configuration=config) + assert apply(old).revision == EVIDENCE_REVISION + before = old.evidence_path.read_bytes() + upgraded = SQLiteMigrationRunner(database, current, configuration=config) + plan = upgraded.plan() + assert plan.revisions == (PROJECTION_REVISION,) + assert plan.domain_steps == ("atomic_memory_evidence_projection",) + assert old.evidence_path.read_bytes() == before + assert ( + upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == PROJECTION_REVISION + ) + assert upgraded.verify().state == "ready" + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence").fetchone() == (0,) + assert connection.execute("SELECT schema_version FROM pc_atomic_memory_evidence_schema").fetchall() == [ + ("p0005",) + ] diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index c32a8fe1a7..3f8c67cabd 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -73,8 +73,8 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_001" - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_5_0_001") + assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_002" + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_5_0_001", "v1_5_0_002") assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index aee3ae603e..d48af8f765 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -561,7 +561,7 @@ def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( ) assert runner.plan(backup_policy="manual").source_revision == "s1300" result = _apply(runner, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_002" assert runner.verify().state == "ready" prepare_server_storage(settings) assert not _apply(runner, policy="manual").changed @@ -692,7 +692,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_002", "dedicated", b'{"history":"unchanged"}') (main_path,) = backend.evidence_directory.glob("*/maintenance.json") evidence = json.loads(main_path.read_text()) @@ -702,7 +702,7 @@ def inspect_history(connection, identity, guard): for inspect in (runner.plan, runner.verify, runner.apply): with pytest.raises(MigrationError, match="checksum_conflict"): inspect() - assert backend.connections().run(inspect_history) == ("v1_5_0_001", "dedicated", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_002", "dedicated", b'{"history":"unchanged"}') assert { path: path.read_bytes() for path in backend.evidence_directory.rglob("maintenance.json") } == original_evidence @@ -769,7 +769,7 @@ def inspect_history(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_history) == ("v1_5_0_001", b'{"history":"unchanged"}') + assert backend.connections().run(inspect_history) == ("v1_5_0_002", b'{"history":"unchanged"}') def test_mysql_registered_projection_repair_preserves_authority_and_revision(backend: _BackendTarget) -> None: @@ -837,7 +837,7 @@ def inspect_content(connection, identity, guard): ).scalar_one(), ) - assert backend.connections().run(inspect_content) == ("v1_5_0_001", content, "Preserve immutable nebula evidence.") + assert backend.connections().run(inspect_content) == ("v1_5_0_002", content, "Preserve immutable nebula evidence.") @pytest.mark.parametrize("origin", ("empty", "released")) @@ -992,7 +992,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan _load_full_server_baseline(backend) runner = _full_server_runner(backend) original = runner.plan(backup_policy="manual") - assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_001" + assert original.source_revision == "s1300" and original.target_revision == "v1_5_0_002" _interrupt_after_ddl(runner, interruption) journals = list(backend.evidence_directory.rglob("maintenance.json")) assert len(journals) == 1 @@ -1014,7 +1014,7 @@ def test_mysql_full_server_interruption_preserves_history_and_original_maintenan result = _apply(resumed, policy="manual") - assert result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.readiness_scope == "server" and result.revision == "v1_5_0_002" assert result.backup_state == "user_confirmed" assert result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" @@ -1094,7 +1094,7 @@ def client_active(connection: Connection | None, _identity: BackendIdentity, _gu # interruption. An arbitrary remote failure cannot be acknowledged blindly. resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_001" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_002" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journals[0].read_text(encoding="utf-8")) diff --git a/tests/builtin/persistence/test_owner_recovery_migration.py b/tests/builtin/persistence/test_owner_recovery_migration.py index e91b321acb..e5be5670e9 100644 --- a/tests/builtin/persistence/test_owner_recovery_migration.py +++ b/tests/builtin/persistence/test_owner_recovery_migration.py @@ -18,6 +18,7 @@ import shutil import sqlite3 from contextlib import closing +from pathlib import Path import pytest @@ -39,13 +40,36 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks shutil.copytree(current.directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - resources = manifest["revision_resources"].pop("v1_4_1_004") - for resource in (*resources, "versions/v1_4_1_004_owner_recovery.py", "mysql/versions/v1_4_1_004.py"): + appended = current.revisions[current.revisions.index("v1_4_1_004") :] + removed_resources = {resource for revision in appended for resource in manifest["revision_resources"].pop(revision)} + removed_steps = [step for step in manifest["domain_steps"] if step["revision"] in appended] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in appended] + for step in removed_steps: + removed_resources.update((step["module"], *step.get("resources", ()))) + retained_resources = {resource for resources in manifest["revision_resources"].values() for resource in resources} + for step in manifest["domain_steps"]: + retained_resources.update((step["module"], *step.get("resources", ()))) + for name in appended: + revision = current.script.get_revision(name) + removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed_resources.add(f"mysql/versions/{name}.py") + for resource in removed_resources - retained_resources: (directory / resource).unlink() - del manifest["sqlite_fingerprints"]["v1_4_1_004"] - del manifest["domain_readiness"]["v1_4_1_004"] - manifest["mysql_state_layouts"].remove("schemas/mysql-v1_4_1_004.json") - manifest["affected_objects"].remove("pc_access_owner_recovery") + for revision in appended: + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest.get("mysql_wrappers", {}).pop(revision, None) + manifest["mysql_state_layouts"] = [ + resource for resource in manifest["mysql_state_layouts"] if resource not in removed_resources + ] + appended_tables = { + "pc_access_owner_recovery", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", + "pc_source_evidence_bindings", + } + manifest["affected_objects"] = [name for name in manifest["affected_objects"] if name not in appended_tables] manifest_path.write_text(json.dumps(manifest), encoding="utf-8") previous = MigrationBundle(directory, embedding_profile=embedding) database = tmp_path / "managed.db" @@ -59,8 +83,8 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration(embedding)) plan = upgraded.plan() - assert plan.source_revision == "s1402" and plan.revisions == ("v1_4_1_004",) - assert apply(upgraded).revision == "v1_4_1_004" + assert plan.source_revision == "s1402" and plan.revisions == appended + assert apply(upgraded).revision == current.head assert upgraded.verify().state == "ready" assert not apply(upgraded).changed evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) @@ -76,8 +100,9 @@ def test_registered_owner_recovery_upgrade_preserves_history_and_revision_checks assert upgraded.plan().schema_fingerprint == fresh.plan().schema_fingerprint +@pytest.mark.parametrize("target", ["owner", "current"]) @pytest.mark.parametrize("damage", [None, "payload-collation", "pending-index"]) -def test_owner_recovery_mysql_registration_matches_native_character_and_key_definitions(damage): +def test_owner_recovery_mysql_registration_matches_native_character_and_key_definitions(damage, target): # Simulated SHOW output checks normalization, not execution on a native backend. definition = """CREATE TABLE `pc_access_owner_recovery` ( `operation_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, @@ -89,8 +114,9 @@ def test_owner_recovery_mysql_registration_matches_native_character_and_key_defi UNIQUE KEY `uq_pc_access_owner_recovery_pending` (`deployment_id`,`completed`,`operation_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci""" bundle = server_bundle() + target_revision = "v1_4_1_004" if target == "owner" else bundle.head inventory = {"tables": {}, "views": [], "triggers": []} - for revision in bundle.revisions[:-1]: + for revision in bundle.revisions[: bundle.revisions.index(target_revision) + 1]: path = bundle.directory / f"schemas/mysql-{revision}.json" fragment = json.loads(path.read_text(encoding="utf-8")) for change in fragment["revisions"][revision]["changes"]: @@ -109,8 +135,8 @@ def test_owner_recovery_mysql_registration_matches_native_character_and_key_defi table["indexes"] = {} schema = MySQLSchema(bundle) if damage is None: - schema.verify(inventory, "v1_4_1_004") - assert schema.identify(inventory) == "v1_4_1_004" + schema.verify(inventory, target_revision) + assert schema.identify(inventory) == target_revision else: with pytest.raises(MigrationError, match="incompatible_schema"): - schema.verify(inventory, "v1_4_1_004") + schema.verify(inventory, target_revision) diff --git a/tests/builtin/persistence/test_portable_storage_migration.py b/tests/builtin/persistence/test_portable_storage_migration.py index 335b998f55..6021e446e3 100644 --- a/tests/builtin/persistence/test_portable_storage_migration.py +++ b/tests/builtin/persistence/test_portable_storage_migration.py @@ -114,8 +114,8 @@ def test_registered_s1401_upgrade_preserves_completed_evidence_and_matches_fresh upgraded = SQLiteMigrationRunner(database, bundle, configuration=config) plan = upgraded.plan() assert plan.source_revision == "s1401" - assert plan.revisions == ("s1402", "v1_4_1_004", "v1_5_0_001") - assert apply(upgraded).revision == "v1_5_0_001" + assert plan.revisions == ("s1402", "v1_4_1_004", "v1_5_0_001", "v1_5_0_002") + assert apply(upgraded).revision == "v1_5_0_002" assert upgraded.verify().state == "ready" assert not apply(upgraded).changed fresh = SQLiteMigrationRunner(tmp_path / "fresh.db", bundle, configuration=config) diff --git a/tests/builtin/persistence/test_release140_server_migrations.py b/tests/builtin/persistence/test_release140_server_migrations.py index d46510a15d..fc0a2c6c87 100644 --- a/tests/builtin/persistence/test_release140_server_migrations.py +++ b/tests/builtin/persistence/test_release140_server_migrations.py @@ -134,7 +134,7 @@ def test_registered_sources_choose_distinct_lineages(tmp_path, release, expected _settings, _embedding, runner = _source(tmp_path, release=release) plan = runner.plan() assert plan.source_revision == expected - assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_5_0_001") + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_5_0_001", "v1_5_0_002") @pytest.mark.parametrize(("vectors", "code"), [(False, False), (True, False), (False, True), (True, True)]) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 5d674241d1..dd60861c1a 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -40,7 +40,10 @@ open_builtin_contexts, open_builtin_runtime, ) -from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.runtime.atomic_memory_rebuild import ( + rebuild_atomic_memory_projection, + verify_atomic_memory_current, +) from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION @@ -254,6 +257,11 @@ async def scenario() -> None: (refs[1], asserted), (refs[2], remote), ) + async with contexts.database.transaction() as connection: + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, "scope", original.ref) + == annotation + ) def reject_snapshot(connection, cursor, statement, *args): if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): @@ -299,11 +307,27 @@ async def assert_retained(contexts, historical): await assert_retained(contexts, annotation) async with contexts.database.transaction() as connection: await connection.execute(text("DELETE FROM pc_atomic_memory_current")) + await connection.execute(text("DELETE FROM pc_atomic_memory_evidence")) + damaged = await verify_atomic_memory_current( + connection, contexts.atomic_memory.index, require_evidence_projection=True + ) + assert not damaged.ready + assert damaged.counts["evidence_projection_differences"] == 3 report = await rebuild_atomic_memory_projection( contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 ) assert report.ready, report.errors await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + for scope_id, reference, expected in ( + ("scope", original.ref, annotation), + ("scope", revised.ref, current), + ("other", other.ref, other_annotation), + ): + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, scope_id, reference) + == expected + ) async with contexts.database.transaction() as connection: await connection.execute( text( @@ -330,6 +354,11 @@ async def assert_retained(contexts, historical): ) assert report.ready, report.errors await assert_retained(contexts, neutral) + async with contexts.database.transaction() as connection: + assert ( + await contexts.atomic_memory.index.read_evidence_projection(connection, "scope", original.ref) + == neutral + ) asyncio.run(scenario()) diff --git a/tests/e2e/test_managed_archive_recovery.py b/tests/e2e/test_managed_archive_recovery.py index 5e8344734d..6589cbe685 100644 --- a/tests/e2e/test_managed_archive_recovery.py +++ b/tests/e2e/test_managed_archive_recovery.py @@ -59,7 +59,7 @@ async def scenario(): ) artifact = await source.records.create_artifact(scope.scope_id, family, ArtifactWrite(content=content)) authoritative = await source.records.get_artifact(scope.scope_id, family, artifact.artifact_id) - await source.portability.export([scope.scope_id], archive, authorize=authorize) + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) async with open_builtin_contexts(config, embedding_model=embedding) as target: with pytest.raises(RuntimeError, match="interrupted projection rebuild"): await PortableBundleService( @@ -86,6 +86,10 @@ async def scenario(): async with open_builtin_contexts(config, embedding_model=embedding) as target: restored = await target.records.get_artifact(scope.scope_id, family, artifact.artifact_id) assert restored.content == authoritative.content + recovered = await target.portability.export( + [scope.scope_id], tmp_path / "recovered.pcb", authorize=authorize + ) + assert recovered.total_digest == exported.total_digest asyncio.run(scenario()) assert ( diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 2f78ef22c8..474c8ebfd8 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -56,7 +56,10 @@ AtomicMemorySearchChannels, AtomicMemorySearchRequest, ) -from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table +from powercontext.builtin.persistence.atomic_memory_index_schema import ( + ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE, + atomic_memory_current_table, +) from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinConfig, CaptureSource, open_builtin_runtime from powercontext.builtin.runtime.artifact_processing import ( @@ -237,7 +240,7 @@ class _VectorMemoryIndex: capabilities = AtomicMemoryIndexCapabilities(fts=False, vector=True, embedding_profile=_VECTOR_PROFILE) table = atomic_memory_current_table() - tables = () + tables = (ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE,) def __init__(self, profile=None): self._evidence = AtomicMemoryEvidenceRepository() @@ -251,6 +254,12 @@ async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): async def read_evidence(self, connection, scope_id, artifact_ref, /): return await self._evidence.read(connection, scope_id, artifact_ref) + async def project_evidence(self, connection, annotation, /): + await self._evidence.replace_projection(connection, annotation) + + async def read_evidence_projection(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read_projection(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /): pass diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 8b60876faa..0c3ae668e1 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -138,6 +138,13 @@ async def scenario() -> None: async with open_builtin_contexts(target_config, source_registry=registry) as target: actual = await target.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) assert actual == expected + async with target.database.transaction() as connection: + assert ( + await target.atomic_memory_index.read_evidence_projection( + connection, scope.scope_id, expected.artifact_ref + ) + == expected + ) asyncio.run(scenario()) diff --git a/tests/native/test_oceanbase_engine_recovery.py b/tests/native/test_oceanbase_engine_recovery.py index 12b189c72b..b2be10e101 100644 --- a/tests/native/test_oceanbase_engine_recovery.py +++ b/tests/native/test_oceanbase_engine_recovery.py @@ -175,7 +175,7 @@ def ping(connection, _identity, _guard): _apply(_full_server_runner(backend), policy="manual") resumed = _full_server_runner(backend, acknowledge_previous_run=token) result = _apply(resumed, policy="manual") - assert result.state == "ready" and result.revision == "v1_5_0_001" and result.readiness_scope == "server" + assert result.state == "ready" and result.revision == "v1_5_0_002" and result.readiness_scope == "server" assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] assert resumed.verify().state == "ready" completed = json.loads(journal.read_text()) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 75a2517134..53c46126ad 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -401,7 +401,7 @@ def test_native_public_migration_switches_an_installed_130_service( if failure in {"startup", "switch"}: assert applied.exit_code == 1, applied.output assert result["error"] == "service_start_failed" - assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_001" + assert result["database_state"] == "ready" and result["revision"] == "v1_5_0_002" assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} phase = "switching" if failure == "switch" else "start_failed" assert json.loads(controller.maintenance_path.read_text())["phase"] == phase diff --git a/tests/test_managed_atomic_maintenance.py b/tests/test_managed_atomic_maintenance.py index 41f968cbbd..dcdb1f2262 100644 --- a/tests/test_managed_atomic_maintenance.py +++ b/tests/test_managed_atomic_maintenance.py @@ -76,7 +76,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) assert plan["source_revision"] == "s1300" - assert plan["target_revision"] == "v1_5_0_001" + assert plan["target_revision"] == "v1_5_0_002" assert plan["readiness_scope"] == "server" assert historical.read_bytes() == original @@ -103,7 +103,7 @@ def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_b with closing(sqlite3.connect(result["backup_ref"])) as connection: assert tuple(connection.iterdump()) == original_dump with closing(sqlite3.connect(historical)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_002",)] assert connection.execute("SELECT content FROM pc_artifacts").fetchall() == [(b'{"history":"unchanged"}',)] verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) assert verified.exit_code == 0, verified.output diff --git a/tests/test_managed_processing_maintenance.py b/tests/test_managed_processing_maintenance.py index b803b5ada9..b7620e50b1 100644 --- a/tests/test_managed_processing_maintenance.py +++ b/tests/test_managed_processing_maintenance.py @@ -75,7 +75,7 @@ def test_registered_mode_switch_backs_up_preserves_revision_and_fences_old_worke assert result.backup_state == "completed" assert result.backup_ref and Path(result.backup_ref).is_file() with closing(sqlite3.connect(path)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_002",)] assert ( json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ "mode" diff --git a/tests/test_migration_revision_names.py b/tests/test_migration_revision_names.py index 42ada4163d..c46c39f990 100644 --- a/tests/test_migration_revision_names.py +++ b/tests/test_migration_revision_names.py @@ -60,9 +60,10 @@ def invoke(action, *options): "v1_4_1_003", "v1_4_1_004", "v1_5_0_001", + "v1_5_0_002", ] assert [item["revision"] for item in plan["pending_migrations"]] == list(reviewed.revisions) - target = {"revision": "v1_5_0_001", "name": "v1_5_0_001", "software_version": "1.5.0", "sequence": 1} + target = {"revision": "v1_5_0_002", "name": "v1_5_0_002", "software_version": "1.5.0", "sequence": 2} assert plan["target_migration"] == target applied = invoke( @@ -77,7 +78,7 @@ def invoke(action, *options): ) assert applied["migration"] == target and applied["server_ready"] with closing(sqlite3.connect(runner.database)) as connection: - assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_001",)] + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("v1_5_0_002",)] evidence = runner.evidence_path.read_bytes() recorded = json.loads(evidence) assert recorded["source_revision"] == expected_source["revision"] @@ -88,6 +89,7 @@ def invoke(action, *options): "s1402", "v1_4_1_004", "v1_5_0_001", + "v1_5_0_002", } assert runner.verify().state == "ready" assert invoke("verify")["migration"] == target diff --git a/tests/test_registered_source_maintenance_cli.py b/tests/test_registered_source_maintenance_cli.py index c593e6c0c4..2fa5251866 100644 --- a/tests/test_registered_source_maintenance_cli.py +++ b/tests/test_registered_source_maintenance_cli.py @@ -63,7 +63,7 @@ def test_release140_compatibility_commands_preserve_source_and_share_full_upgrad planned = CliRunner().invoke(cli, [*arguments, "plan", "--backup", "skip"]) assert planned.exit_code == 0, planned.output plan = json.loads(planned.output) - assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_001" + assert plan["source_revision"] == "r1400" and plan["target_revision"] == "v1_5_0_002" assert plan["readiness_scope"] == "server" full = CliRunner().invoke(cli, ["server", "db-migrate", "plan", "--backup", "skip"]) assert full.exit_code == 0, full.output From 34c5276f147d253f38bd0774ab36aaf5870a415d Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 28 Oct 2026 14:50:08 +0800 Subject: [PATCH 23/26] fix(memory): persist versioned Source evidence and close review findings Retain exact Atomic Memory evidence declarations and bindings for inspectable, rebuildable review. Declarations come only from trusted adapters or deployment configuration; missing or unverifiable records stay neutral. Close the review findings on this chain: resume registered MySQL DDL prefixes before the final evidence marker, bound evidence lookups to SQLite variable limits, reuse native manifests only within one accepted batch, match populated optional and nested observation schemas to worker serialization including covered unions, and keep deliberate neutrality distinct from digest corruption. --- docs/en/docs/operate/database-migrations.md | 29 ++- docs/en/docs/operate/observability.md | 8 + docs/en/docs/workflows/sources.md | 19 ++ docs/zh/docs/operate/database-migrations.md | 28 ++- docs/zh/docs/operate/observability.md | 8 + docs/zh/docs/workflows/sources.md | 19 ++ release/upgrade-support.json | 4 +- src/powercontext/__init__.py | 2 + .../persistence/atomic_memory_evidence.py | 72 ++++--- .../migrations/atomic_memory_evidence.py | 14 +- .../migrations/atomic_projection.py | 3 + .../atomic_memory_evidence.py | 14 +- .../atomic_memory_evidence_projection.py | 14 +- .../builtin/persistence/source_definitions.py | 10 +- .../builtin/persistence/sources.py | 164 ++++++++++------ src/powercontext/builtin/runtime/__init__.py | 2 + .../builtin/runtime/application.py | 6 + .../builtin/runtime/atomic_memory_rebuild.py | 30 ++- .../builtin/runtime/composition.py | 3 + src/powercontext/errors.py | 11 ++ src/powercontext/server/cli.py | 3 + src/powercontext/server/logging.py | 3 +- src/powercontext/sources/definitions.py | 7 +- src/powercontext/sources/evidence.py | 45 +++++ src/powercontext/sources/observations.py | 65 ++++++- .../test_atomic_evidence_schema_copies.py | 35 ++++ .../test_evidence_neutralization_logging.py | 166 +++++++++++++++++ .../test_mysql_initialization_preflight.py | 124 ++++++++++++ .../test_mysql_migration_runner.py | 11 +- tests/builtin/persistence/test_sources.py | 89 +++++++++ .../e2e/test_atomic_memory_source_evidence.py | 56 ++++++ tests/test_server_logging.py | 2 + tests/test_source_observations.py | 176 +++++++++++++++++- 33 files changed, 1108 insertions(+), 134 deletions(-) create mode 100644 tests/builtin/persistence/test_atomic_evidence_schema_copies.py create mode 100644 tests/builtin/persistence/test_evidence_neutralization_logging.py diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index b679113b19..8ccb93df9c 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -16,8 +16,8 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 The full Server framework registers two installed-release sources for the current target (stored compatibility IDs shown below): -- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001`. -- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001` through a separate frozen source bridge. +- **1.3.0:** `s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001 → v1_5_0_002`. +- **1.4.0:** `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001 → v1_5_0_002` through a separate frozen source bridge. Application release numbers and schema revisions are distinct. The 1.4.0 release is not the `s1400` checkpoint: its existing Candidate schema is current, but its Topic attempt storage still needs the registered bridge. @@ -53,6 +53,13 @@ Revision `v1_4_1_004` registers `pc_access_owner_recovery` for durable Artifact Revision `v1_5_0_001` adds the Source evidence bindings, exact Atomic Memory evidence snapshots, and evidence schema marker. Its own domain step validates retained Source material and historical evidence for both lexical and vector profiles. Earlier projection validators and their recorded revision checksums remain unchanged; evidence validation is required only from this revision onward. Existing Sources and Memory revisions without accepted declarations stay neutral (`untrusted` / `unknown`); upgrading never backfills trust. Current projection repair and rebuild also require evidence validation. +Revision `v1_5_0_002` registers `pc_atomic_memory_evidence`, the disposable exact-revision evidence projection. It is +derived from retained history, can be discarded and rebuilt, and never becomes the authority for a trust decision. + +The runtime bundle stamps `pc_atomic_memory_evidence_schema.schema_version` with `p0005`, its own revision for the same +tables; the Server chain registers that shape as `v1_5_0_001`. Operator messages name both so a runtime deployment and a +managed Server deployment can each apply the revision they own. + An interrupted [portable restore](portable-archive.md) keeps normal startup and workers blocked until its projections are complete. Retry `powercontext archive restore --yes` with the same validated bundle. Only this explicit recovery path can open an incomplete projection set, after validating the registered layout and the configured vector @@ -101,12 +108,13 @@ Sequence `000` identifies a registered historical baseline; upgrade steps start | `v1_4_1_003` | 1.4.1 | 3 | `s1402` | Portable restore receipts and timestamp adaptation | | `v1_4_1_004` | 1.4.1 | 4 | `v1_4_1_004` | Durable external-owner recovery intents | | `v1_5_0_001` | 1.5.0 | 1 | `v1_5_0_001` | Retained Source evidence and historical validation | +| `v1_5_0_002` | 1.5.0 | 2 | `v1_5_0_002` | Disposable exact-revision evidence projection | -Both registered source paths reach `v1_5_0_001`; only 1.3.0 is selected for CI upgrade acceptance: +Both registered source paths reach `v1_5_0_002`; only 1.3.0 is selected for CI upgrade acceptance: ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -> v1_5_0_002 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -> v1_5_0_002 ``` `status` and `plan` show `source_migration`, `target_migration` and `pending_migrations` @@ -128,7 +136,7 @@ existing plans and original backups usable. Partial prototype bundles keep their For a new Server migration, use the release-based name directly as its Alembic `revision` and register its schema, domain steps and recovery checkpoints as usual. `down_revision` must reference -the actual preceding ID (`v1_5_0_001` when appending directly to the current head), even if that ID has +the actual preceding ID (`v1_5_0_002` when appending directly to the current head), even if that ID has a display alias. The framework follows this registered chain, not lexical name ordering. Already registered names and release assignments stay immutable; a version bump must not rename history. @@ -223,6 +231,11 @@ bound row buffering; this is an offline whole-projection operation. It does not embedded seekDB Code configurations. The compatibility command `server atomic-memory-rebuild-projection` delegates managed databases to this bundle and supports `--action plan/apply/verify` with the same confirmation and backup options. +Both maintenance entry points run from the migration bundle and therefore admit evidence against the builtin Source +registry; they do not receive a host's code-supplied `SourceDefinitionRegistry`. The runtime evidence read path +(`AtomicMemory.evidence`) uses the configured registry, so a deployment with custom native Definitions should rebuild +through the Python API with `sources=` when it needs an identical verdict. + ## Topic Memory recovery bundle Select `--bundle topic-memory-recovery` for the versioned `powercontext.topic-memory-recovery.v1` conversion. It recognizes @@ -620,6 +633,10 @@ SQL literals reach the backend unchanged. Repositories run after schema checks. Both full Server source chains register exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. +During `v1_5_0_001`, an interrupted MySQL-compatible upgrade can resume a registered CREATE-table prefix before the +final `p0005` evidence marker is inserted. Full evidence validation starts once that marker exists; a completed +database with missing or incompatible evidence storage still fails admission. + Online readiness reads Atomic Memory authority and projections from the same snapshot without locking business head rows. Offline projection repair retains its locked state comparison and write-side concurrency protection. The 1.4.0 bridge is assembled from immutable shared resources and its `server_sources/1.4.0` overlay. Source diff --git a/docs/en/docs/operate/observability.md b/docs/en/docs/operate/observability.md index cac2c44cb3..bdc8a0c16b 100644 --- a/docs/en/docs/operate/observability.md +++ b/docs/en/docs/operate/observability.md @@ -63,6 +63,14 @@ including code stages. Code-specific diagnostic fields are included only for `co | `worker_id`, `family`, `binding`, `request_generation`, `supervisor_group`, `supervisor_generation` | One worker attempt and its accepted work/fence; correlate child output with supervisor diagnostics | | `duration_ms`, `error_code`, `error_category` | Duration in milliseconds, diagnostic code, and bounded operational error classification | +`reason` is a bounded cause for a retained-evidence downgrade. The `atomic_memory.evidence.neutral` event reports that +accepted evidence could not be attested and therefore resolved to `untrusted` / `unknown`; it never carries material, +declaration bodies, or requested content. Its `unit` is the retained fact (`source`, `declaration`, `manifest`, or +`revision`), `outcome` is `neutral`, and `reason` is one of `schema_mismatch`, `digest_mismatch`, `no_manifest`, +`decode_failed`, `legacy_unverifiable`, `invalid_declaration`, `manifest_conflict`, `definition_missing`, or +`evidence_mismatch`. Integrity causes are emitted at `WARNING`; expected historical gaps stay at `INFO` or `DEBUG`. +Integrity causes such as `manifest_conflict` are alertable and distinct from an absent declaration. + Server roles are `all` (API and supervisor), `api`, and `background`; child processes use `worker`. Explicit logging configuration without a deployment role uses `server`. JSON identity fields correspond to trace resource attributes `service.name`, `service.version`, `service.instance.id`, and `powercontext.service.role`. `service_version` is absent diff --git a/docs/en/docs/workflows/sources.md b/docs/en/docs/workflows/sources.md index 6874f1fb0d..7c6eebc5f0 100644 --- a/docs/en/docs/workflows/sources.md +++ b/docs/en/docs/workflows/sources.md @@ -56,8 +56,27 @@ Remote Source observations configure the same declaration in trusted deployment name, version, and manifest fingerprint. A missing, invalid, or unavailable declaration resolves to `untrusted`/`unknown`; it does not block capture, processing, or existing retrieval. +Observation manifests validate the worker's serialized payload, including populated optional fields and nested models. +Validation aliases do not require workers to rename serialized keys, including when a model forbids extra fields. +For unions, an alias-renamed branch may retain the validation manifest when another branch has an equivalent flat +primitive-property schema covering its serialized output. Unproven branch coverage uses the serialization schema. + Each Memory revision records the declaration and exact Source materialization accepted with it. That retained evidence is authoritative and versioned. The index projection beside it is derived: maintenance can discard and rebuild it without consulting today's registry. Declarations add no ranking weight, score, or access right. +Read the retained facts back for one exact revision: + +```python +from powercontext.builtin.runtime import AtomicMemoryEvidenceAnnotation + +memory = runtime.atomic_memory.for_scope(scope_id) +annotation: AtomicMemoryEvidenceAnnotation = await memory.evidence(artifact_id, revision=3) +for source in annotation.sources: + print(source.source_ref.source_type, source.declaration.authority, source.declaration.verification) +``` + +A declaration that can no longer be attested resolves to `untrusted`/`unknown`. A record that no longer matches its +exact Source material raises `AtomicMemoryEvidenceError`, which callers catch through the `PowerContextError` family. + Source writes and reads are subject to the selected [Scope and access policy](scopes-and-access.md). diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index b4d31d7cee..f631d5c382 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -14,8 +14,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 完整 Server 框架登记了两个真实安装包来源： -- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001`。 -- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001`。 +- **1.3.0：**`s1300 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001 → v1_5_0_002`。 +- **1.4.0：**通过独立冻结的来源桥接，执行 `r1400 → s1400 → s1401 → s1402 → v1_4_1_004 → v1_5_0_001 → v1_5_0_002`。 应用发布版本与 schema revision 分别编号。1.4.0 发行包不等于 `s1400` 检查点：其 Candidate 结构已经是当前格式， 但 Topic 尝试状态仍需登记的桥接。两条路径保留既有引用和历史，到达相同的受管目标结构；原有资源摘要分别保留， @@ -46,6 +46,13 @@ payload 身份及待处理 intent。兼容的当前任务原样保留，不套 修订 `v1_5_0_001` 新增 Source 证据绑定、精确 Atomic Memory 证据快照和证据结构标记。它使用独立领域步骤验证保留的 Source 材料与历史证据，覆盖词法和向量配置。早期投影验证器及已记录的修订校验和保持不变；仅从此修订开始要求证据验证。既有 Source 和 Memory 修订若未接受声明，仍保持中性（`untrusted` / `unknown`），升级不会回填信任。当前投影修复与重建也必须验证证据。 +修订 `v1_5_0_002` 登记 `pc_atomic_memory_evidence`，即精确 revision 的可丢弃证据投影。它由保留历史派生， +可以丢弃并重建，从不作为信任判定的权威依据。 + +runtime bundle 在 `pc_atomic_memory_evidence_schema.schema_version` 中写入 `p0005`，即它自身应用同一结构所用的 +revision；Server 链将同一结构登记为 `v1_5_0_001`。运维提示会同时给出两者，使 runtime 部署与受管 Server 部署 +各自应用所属的 revision。 + [可移植归档恢复](portable-archive.md) 中断后，普通启动和 worker 会继续拒绝未完成的检索投影。 请使用同一个通过校验的归档，重新执行 `powercontext archive restore --yes`。 仅此显式恢复路径可在验证已登记结构和配置的向量 profile 后打开投影尚未完整的数据库，且不启动 worker。 @@ -86,12 +93,13 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `v1_4_1_003` | 1.4.1 | 第三步 | `s1402` | 可移植恢复回执与时间戳适配 | | `v1_4_1_004` | 1.4.1 | 第四步 | `v1_4_1_004` | 持久外部所有权恢复意图 | | `v1_5_0_001` | 1.5.0 | 第一步 | `v1_5_0_001` | 保留 Source 证据及历史验证 | +| `v1_5_0_002` | 1.5.0 | 第二步 | `v1_5_0_002` | 可丢弃的精确 revision 证据投影 | -两条已登记来源路径都升级到 `v1_5_0_001`；CI 升级验收只选择 1.3.0： +两条已登记来源路径都升级到 `v1_5_0_002`；CI 升级验收只选择 1.3.0： ```text -1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 +1.3.0: v1_3_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -> v1_5_0_002 +1.4.0: v1_4_0_000 -> v1_4_1_001 -> v1_4_1_002 -> v1_4_1_003 -> v1_4_1_004 -> v1_5_0_001 -> v1_5_0_002 ``` `status` 和 `plan` 在原有机器字段之前显示 `source_migration`、`target_migration`、 @@ -110,7 +118,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 这样可以继续使用已有计划和原始备份。局部原型 bundle 保留自己的独立编号。 后续新增 Server 迁移直接使用该命名作为 Alembic `revision`，并正常登记结构、领域步骤和恢复检查点。 -`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_001`，即使它已有展示别名。 +`down_revision` 必须引用前一步的实际 ID；若直接追加在当前 head 之后，引用 `v1_5_0_002`，即使它已有展示别名。 框架按照登记的链执行，不按名称字典序推算。已登记的名称和所属发布版本保持不变，版本升级不能重命名历史。 ## 初始化与维护职责 @@ -194,6 +202,11 @@ apply 只修复派生行，完整 Server 校验通过后才提交。权威数据 兼容命令 `server atomic-memory-rebuild-projection` 在受管数据库上转发到此 bundle， 支持 `--action plan/apply/verify` 和同样的确认、备份选项。 +这两个维护入口均由迁移 bundle 构建，因此证据准入使用内置 Source registry，拿不到宿主以代码传入的 +`SourceDefinitionRegistry`。运行时的证据读取路径（`AtomicMemory.evidence`）使用已配置的 registry； +若部署注册了自定义 native Definition 且需要一致的判定结果，应通过 Python API 传入 +`sources=` 执行重建。 + ## Topic Memory 恢复 bundle 选择 `--bundle topic-memory-recovery` 执行版本化的 `powercontext.topic-memory-recovery.v1` 转换。它识别恢复功能之前的 @@ -453,6 +466,9 @@ MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `m Repository 仅在结构满足后运行。两条完整 Server 来源链均用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 +在 `v1_5_0_001` 中，MySQL 兼容升级若在最终 `p0005` evidence 标记写入前中断，可以从已登记的建表前缀继续。 +标记存在后才要求完整 evidence 校验；已完成的数据库若缺失 evidence 存储或结构不兼容，仍会被准入检查拒绝。 + 在线就绪校验在同一快照中读取 Atomic Memory 权威记录与投影，不锁定业务 head 行。离线投影修复仍保留加锁状态比较和写入并发保护。 1.4.0 桥接由不可改写的共享资源和 `server_sources/1.4.0` 来源覆盖层组装。执行器根据完整未登记版本的数据库 diff --git a/docs/zh/docs/operate/observability.md b/docs/zh/docs/operate/observability.md index 2267240326..73a2d1a37d 100644 --- a/docs/zh/docs/operate/observability.md +++ b/docs/zh/docs/operate/observability.md @@ -61,6 +61,14 @@ Code 专用诊断字段仅在 `code.*` events 中输出。 | `worker_id`、`family`、`binding`、`request_generation`、`supervisor_group`、`supervisor_generation` | 一次 worker attempt 及其已接受的工作和 fence，用于关联 child 与 supervisor logs | | `duration_ms`、`error_code`、`error_category` | 毫秒耗时、诊断 code 和有限取值的 operational error category | +`reason` 是保留证据降级的有限原因。`atomic_memory.evidence.neutral` 事件表示已接受的证据无法被佐证， +因此解析为 `untrusted` / `unknown`；该事件不携带材料、声明正文或请求内容。其 `unit` 为保留事实的类型 +（`source`、`declaration`、`manifest` 或 `revision`），`outcome` 为 `neutral`，`reason` 取 +`schema_mismatch`、`digest_mismatch`、`no_manifest`、`decode_failed`、`legacy_unverifiable`、 +`invalid_declaration`、`manifest_conflict`、`definition_missing` 或 `evidence_mismatch` 之一。 +已登记的完整性原因以 `WARNING` 输出；可预期的历史缺口保持 `INFO` 或 `DEBUG`。`manifest_conflict` 这类完整性原因 +可告警，并与“从未声明”区分开。 + Server role 为 `all`（API 与 supervisor）、`api` 或 `background`，child process 使用 `worker`。 显式配置 logging 但没有指定 deployment role 时使用 `server`。JSON identity 字段对应 trace resource attributes： `service.name`、`service.version`、`service.instance.id` 和 `powercontext.service.role`。 diff --git a/docs/zh/docs/workflows/sources.md b/docs/zh/docs/workflows/sources.md index 380e9f8cd7..d209c67d66 100644 --- a/docs/zh/docs/workflows/sources.md +++ b/docs/zh/docs/workflows/sources.md @@ -53,8 +53,27 @@ definition = AdapterSourceDefinition( 远程 Source observation 在受信任的部署代码中配置同一声明，键为精确的 Definition 名称、版本与 manifest fingerprint。声明缺失、无效或不可用时解析为 `untrusted`/`unknown`，不阻塞采集、处理或既有检索。 +Observation manifest 校验 worker 序列化后的 payload，包括已填充的可选字段与嵌套模型。 +即使模型禁止额外字段，validation alias 也不要求 worker 重命名序列化后的键。 +对于 Union，若另一分支的等价扁平基础字段 schema 能覆盖 alias 分支的序列化输出，可保留 validation manifest。 +无法证明分支覆盖时，使用 serialization schema。 + 每个 Memory revision 记录当时接受的声明与精确 Source materialization。保留的证据是权威且带版本的；其旁的索引 投影是派生数据，维护可以丢弃它并从保留事实重建，且不查询当前 registry。声明不增加排序权重、不改变分数， 也不授予访问权限。 +按精确 revision 读回保留事实： + +```python +from powercontext.builtin.runtime import AtomicMemoryEvidenceAnnotation + +memory = runtime.atomic_memory.for_scope(scope_id) +annotation: AtomicMemoryEvidenceAnnotation = await memory.evidence(artifact_id, revision=3) +for source in annotation.sources: + print(source.source_ref.source_type, source.declaration.authority, source.declaration.verification) +``` + +无法再被佐证的声明解析为 `untrusted`/`unknown`。若记录与精确 Source 材料不再一致，会抛出 +`AtomicMemoryEvidenceError`，调用方可通过 `PowerContextError` 家族捕获。 + Source 读写受所选 [Scope 与访问策略](scopes-and-access.md)约束。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index de5a4cd808..1d0a0866fc 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "5e2817a293f90457217ce3d6004aeecec5a022fe0f3eef402b53006003f9acd9", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB full-server-code and full-server-vector-code are temporarily manual-only due to native DDL waits with the available engine. Main and publication require the other ten; no manual result is needed for publication. The two deferred configurations remain unverified and are excluded from generated supported upgrade declarations. Reports retain their identities and explicitly record incomplete full-matrix acceptance. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the same automatic matrix, manual-only configurations and publication restrictions. Revision v1_5_0_001 follows v1_4_1_004 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_002 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions." + "persistence_sha256": "e2c4902e432541f23cc38a6c517b3a2f2a20d6e5bb0606c4ea9fb560b125c995", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. Revision v1_5_0_001 follows v1_4_1_004 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_002 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index f7589beabc..91db258bf6 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -28,6 +28,7 @@ ArtifactError, ArtifactFamilyMismatchError, ArtifactNotFoundError, + AtomicMemoryEvidenceError, ConnectorError, ConnectorSubmissionRejectedError, InvalidArtifactReferenceError, @@ -80,6 +81,7 @@ "ArtifactRef", "ArtifactStore", "Artifacts", + "AtomicMemoryEvidenceError", "ConnectorError", "ConnectorSubmissionRejectedError", "InvalidArtifactReferenceError", diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py index 7170a0ce83..614db7b3cc 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -17,8 +17,9 @@ from __future__ import annotations import json +import logging from collections.abc import Sequence -from typing import Literal +from typing import Literal, cast import rfc8785 from aiosqlite import Connection as SQLiteConnection @@ -33,9 +34,14 @@ from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE from powercontext.builtin.persistence.sources import SourceRepository, _require_identity +from powercontext.errors import AtomicMemoryEvidenceError from powercontext.limits import MAX_SCOPE_ID_LENGTH from powercontext.sources import SourceRef -from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence +from powercontext.sources.evidence import ( + MemoryEvidenceDeclaration, + SourceEvidence, + log_evidence_neutralized, +) class AtomicMemoryEvidenceAnnotation(BaseModel): @@ -73,6 +79,7 @@ async def save( try: declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) except ValidationError: + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) declaration = MemoryEvidenceDeclaration() accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) annotation = AtomicMemoryEvidenceAnnotation( @@ -160,7 +167,6 @@ async def verify( No accepted snapshot is saved or repaired by this operation. """ - from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY artifact_ref = self._reference(scope_id, artifact_ref) @@ -172,26 +178,26 @@ async def verify( for accepted in annotation.sources: material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) if material is not None and accepted.material_digest != material.material_digest: - raise AtomicMemoryIndexError( - "evidence-material", "Historical evidence does not match the exact Source material" - ) + raise AtomicMemoryEvidenceError if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( item.source_ref for item in materials ): + log_evidence_neutralized("evidence_mismatch", unit="revision") return self._neutral(scope_id, artifact_ref, materials) - verified = tuple( - accepted + verified = [] + for accepted, material in zip(annotation.sources, materials, strict=True): if ( accepted.materialization == material.materialization and accepted.definition_name == material.definition_name and accepted.definition_version == material.definition_version and accepted.manifest_fingerprint == material.manifest_fingerprint and accepted.declaration == material.declaration - ) - else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for accepted, material in zip(annotation.sources, materials, strict=True) - ) - return annotation.model_copy(update={"sources": verified}) + ): + verified.append(accepted) + continue + log_evidence_neutralized("evidence_mismatch", unit="source", level=logging.WARNING) + verified.append(accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()})) + return annotation.model_copy(update={"sources": tuple(verified)}) @staticmethod def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: @@ -217,24 +223,11 @@ async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: Art @staticmethod def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: try: - if not isinstance(payload, (bytes, bytearray, memoryview)): - return None - values = json.loads(bytes(payload)) - if not isinstance(values, list): - return None - sources = [] - for value in values: - if not isinstance(value, dict): - return None - try: - declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) - except ValidationError: - declaration = MemoryEvidenceDeclaration() - sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) - return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) + return _decode_annotation(scope_id, artifact_ref, payload) except (ValueError, TypeError): # An unreadable declaration snapshot conveys no historical claim. # This does not catch SQL errors or actual Source resolution errors. + log_evidence_neutralized("decode_failed", unit="declaration", level=logging.WARNING) return None @staticmethod @@ -248,3 +241,26 @@ def _neutral( artifact_ref=artifact_ref, sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), ) + + +class _InvalidEvidenceSnapshot(ValueError): + """A retained evidence snapshot is not a decodable record list.""" + + +def _decode_annotation(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation: + """Decode one stored snapshot, raising when it is not a valid record list.""" + + if not isinstance(payload, (bytes, bytearray, memoryview)): + raise _InvalidEvidenceSnapshot + values = json.loads(bytes(payload)) + if not isinstance(values, list) or any(not isinstance(value, dict) for value in values): + raise _InvalidEvidenceSnapshot + sources = [] + for value in cast("list[dict[str, object]]", values): + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) + declaration = MemoryEvidenceDeclaration() + sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py index d23f81fd2e..4e462fb7f3 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -43,7 +43,10 @@ class AtomicMemoryEvidenceSchemaError(PersistenceError): def __init__(self, detail: str) -> None: message = { - "missing": "Atomic Memory evidence schema is incomplete; apply the registered p0005 migration", + "missing": ( + "Atomic Memory evidence schema is incomplete; apply the registered evidence revision " + "(p0005 for runtime bundles, v1_5_0_001 in the Server chain)" + ), "version": "Atomic Memory evidence schema version is unsupported", "shape": "Atomic Memory evidence table has an incompatible shape", }[detail] @@ -103,6 +106,15 @@ async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tab async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: + """Require the registered table shape: exact for SQLite, structural for MySQL. + + SQLite stores the executed CREATE text in ``sqlite_master``, so exact equality + is the check available without a second SQL parser; it also freezes the Server + bundle and the runtime revision to one byte-identical DDL copy, asserted by + ``tests/builtin/persistence/test_atomic_evidence_schema_copies.py``. MySQL + reuses the frozen inventory parser because dumps and replays may normalize + physical formatting. + """ if dialect == "sqlite": actual = await connection.exec_driver_sql( "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " diff --git a/src/powercontext/builtin/persistence/migrations/atomic_projection.py b/src/powercontext/builtin/persistence/migrations/atomic_projection.py index 492b8d8240..c76ec7f8cc 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_projection.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_projection.py @@ -96,6 +96,8 @@ def _index(self, connection): def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str, bool, bool]: check_server_layout(connection) index = self._index(connection) + # Built from the migration bundle, so no host SourceDefinitionRegistry exists: + # admission uses BUILTIN_SOURCE_REGISTRY while host reads use the configured one. try: report = finish( atomic_memory_rebuild.verify_atomic_memory_current( @@ -169,6 +171,7 @@ def _rows_digest(self, connection, current): @override def _apply_operation(self, connection: Connection, guard) -> None: index = self._index(connection) + # No host registry on this bundle path; see the note in _state. report = finish( atomic_memory_rebuild.rebuild_atomic_memory_current( pinned(connection), index, batch_size=self.batch_size, guard=guard diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py index 89826e439a..66f5d7dfa5 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence.py @@ -14,10 +14,10 @@ """Validate retained evidence only after its introducing Server revision.""" -from sqlalchemy import inspect +from sqlalchemy import inspect, select from powercontext.builtin.artifacts.memory import EmbeddingProfile -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE from powercontext.builtin.persistence.migrations.atomic_memory_evidence import require_atomic_memory_evidence_schema from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.pinned import finish, pinned @@ -27,9 +27,13 @@ def preflight(connection, *, configuration): - # Planning earlier revisions must not require tables that their pending DDL creates. - tables = set(inspect(connection).get_table_names()) - if any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + # MySQL may commit the marker table before the remaining DDL. Its row is + # inserted last; only that completed declaration requires full readiness. + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + if ( + marker.name in inspect(connection).get_table_names() + and connection.execute(select(marker.c.schema_version).limit(1)).first() is not None + ): finish(require_atomic_memory_evidence_schema(pinned(connection))) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py index e6b90989cb..cf6f0ce2e5 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/atomic_memory_evidence_projection.py @@ -14,7 +14,10 @@ """Rebuild disposable exact-revision evidence from retained declarations.""" +from sqlalchemy import inspect, select + from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE from powercontext.builtin.persistence.migrations.atomic_memory_evidence import require_atomic_memory_evidence_schema from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.pinned import finish, pinned @@ -38,10 +41,13 @@ def _index(connection, configuration): def preflight(connection, *, configuration): - # Earlier pending revisions may not yet have created retained evidence tables. - from sqlalchemy import inspect - - if "pc_atomic_memory_evidence_schema" in inspect(connection).get_table_names(): + # The marker table may precede other retained tables in a committed MySQL + # DDL prefix. Its final row distinguishes readiness from resumable work. + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + if ( + marker.name in inspect(connection).get_table_names() + and connection.execute(select(marker.c.schema_version).limit(1)).first() is not None + ): finish(require_atomic_memory_evidence_schema(pinned(connection))) diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index beffb3f87f..fe0cddd470 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -16,6 +16,7 @@ from __future__ import annotations +import logging from collections.abc import Mapping from typing import Any @@ -33,6 +34,7 @@ ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE from powercontext.sources import SourceDefinition, SourceDefinitionManifest +from powercontext.sources.evidence import log_evidence_neutralized from powercontext.sources.observations import _manifest_for_definition @@ -89,10 +91,16 @@ async def register_native( try: manifest = _manifest_for_definition(definition, transport=False) except Exception: # Native schema providers may not expose a valid manifest. + log_evidence_neutralized("no_manifest", unit="manifest") return None try: return await self.register(connection, manifest) - except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError, StoredPayloadConflictError): + except StoredPayloadConflictError: + # Two manifests exist under one identity; evidence cannot choose either. + log_evidence_neutralized("manifest_conflict", unit="manifest", level=logging.WARNING) + return None + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) return None async def get( diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 5da26bfdca..4a2df3ce6a 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -16,6 +16,7 @@ from __future__ import annotations +import logging from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress from dataclasses import dataclass @@ -63,6 +64,7 @@ SourceObservation, SourceRef, ) +from powercontext.sources.evidence import log_evidence_neutralized _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -73,6 +75,20 @@ PROFILE_SOURCE_WINDOW_BINDING, ) _SOURCE_LOOKUP_KEYS_PER_STATEMENT = 400 # Two bind values per key, including SQLite's 999-variable builds. +_VALIDATORS: dict[str, Any] = {} + + +def _source_validator(manifest: SourceDefinitionManifest) -> Any: + """Compile a retained schema once per immutable manifest fingerprint.""" + + validator = _VALIDATORS.get(manifest.fingerprint) + if validator is None: + schema = manifest.source_schema + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema, registry=Registry()) + if len(_VALIDATORS) < 128: + _VALIDATORS[manifest.fingerprint] = validator + return validator class _StoredSourcePayload(BaseModel): @@ -211,8 +227,11 @@ async def add_many_with_status( .values(position=position) ) await connection.execute(insert(SOURCES_TABLE), new_rows) + # Reuse only within this journal-locked batch; a rollback or later + # registration must never inherit an earlier transaction's result. + native_manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} for ref, source in new_sources.values(): - await self._bind_memory_evidence(connection, scope_id, ref, source) + await self._bind_memory_evidence(connection, scope_id, ref, source, native_manifests) for (source_type, source_id), payload in upgrades.items(): await connection.execute( update(SOURCES_TABLE) @@ -360,21 +379,25 @@ async def historical_evidence( identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) if not identities: return () - rows = ( - ( - await connection.execute( - select(SOURCES_TABLE).where( - SOURCES_TABLE.c.scope_id == scope_id, - tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_(identities), + by_ref: dict[tuple[str, str], Mapping[Any, Any]] = {} + for offset in range(0, len(identities), _SOURCE_LOOKUP_KEYS_PER_STATEMENT): + rows = ( + ( + await connection.execute( + select(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == scope_id, + tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_( + identities[offset : offset + _SOURCE_LOOKUP_KEYS_PER_STATEMENT] + ), + ) ) ) + .mappings() + .all() ) - .mappings() - .all() - ) - by_ref = {(row["source_type"], row["source_id"]): row for row in rows} + by_ref.update(((row["source_type"], row["source_id"]), row) for row in rows) materials: list[SourceEvidence] = [] - fingerprints: dict[tuple[str, str], str] = {} + fingerprints: dict[tuple[str, str], str | None] = {} for ref in identities: reference = SourceRef(source_type=ref[0], source_id=ref[1]) row = by_ref.get(ref) @@ -382,7 +405,7 @@ async def historical_evidence( raise RepositoryNotFoundError("source", (scope_id, reference)) source, material, verifiable = await self._historical_material(connection, reference, row) if not verifiable: - fingerprints[ref] = "" # Never authorize a retained declaration for this capture. + fingerprints[ref] = None # Deliberately unverifiable, not a fingerprint mismatch. if isinstance(source, SourceObservation): fingerprints[ref] = source.definition_fingerprint elif ref[0] == CONTENT_SOURCE_NAME: @@ -446,15 +469,15 @@ async def _historical_material( manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: try: - Draft202012Validator.check_schema(manifest.source_schema) value = source.payload if isinstance(source, SourceObservation) else material - Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) + _source_validator(manifest).validate(value) except JsonSchemaValidationError as error: if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( material ): # Older releases bound declarations without checking serialization # compatibility. Exact accepted bytes remain readable, but neutral. + log_evidence_neutralized("schema_mismatch", unit="source", level=logging.DEBUG) return source, material, False raise InvalidStoredPayloadError( "source", reference.source_type, "payload does not match the historical schema" @@ -628,6 +651,8 @@ def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSour try: definition = self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: + # No configured Definition can perform the live-decoder cross-check. + log_evidence_neutralized("definition_missing", unit="source", level=logging.DEBUG) return if _claims_identity_keys(definition.source_class): # The retained canonical identity/header is authoritative; a live business alias may claim it. @@ -646,63 +671,67 @@ async def _bound_evidence( connection: AsyncConnection, scope_id: str, materials: tuple[SourceEvidence, ...], - fingerprints: Mapping[tuple[str, str], str], + fingerprints: Mapping[tuple[str, str], str | None], ) -> tuple[SourceEvidence, ...]: if not materials: return () table = SOURCE_EVIDENCE_BINDINGS_TABLE - rows = ( - ( - await connection.execute( - select(table).where( - table.c.scope_id == scope_id, - tuple_(table.c.source_type, table.c.source_id).in_( - tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) - ), + identities = tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) + bindings: dict[tuple[str, str], Mapping[Any, Any]] = {} + for offset in range(0, len(identities), _SOURCE_LOOKUP_KEYS_PER_STATEMENT): + rows = ( + ( + await connection.execute( + select(table).where( + table.c.scope_id == scope_id, + tuple_(table.c.source_type, table.c.source_id).in_( + identities[offset : offset + _SOURCE_LOOKUP_KEYS_PER_STATEMENT] + ), + ) ) ) + .mappings() + .all() ) - .mappings() - .all() - ) - bindings = {(row["source_type"], row["source_id"]): row for row in rows} + bindings.update(((row["source_type"], row["source_id"]), row) for row in rows) manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} evidence: list[SourceEvidence] = [] for value in materials: identity = (value.source_ref.source_type, value.source_ref.source_id) + if identity in fingerprints and fingerprints[identity] is None: + evidence.append(value) + continue binding = bindings.get(identity) - if binding is None or ( + if binding is None: + # Captured before evidence bindings existed; no declaration was accepted. + log_evidence_neutralized("legacy_unverifiable", unit="source", level=logging.DEBUG) + evidence.append(value) + continue + if ( binding["material_digest"] != value.material_digest or binding["definition_name"] != value.definition_name or binding["definition_version"] != value.definition_version - or not isinstance(binding["manifest_fingerprint"], str) + or ( + binding["manifest_fingerprint"] is not None and not isinstance(binding["manifest_fingerprint"], str) + ) ): + log_evidence_neutralized("digest_mismatch", unit="source", level=logging.WARNING) + evidence.append(value) + continue + if binding["manifest_fingerprint"] is None: + log_evidence_neutralized("no_manifest", unit="manifest") evidence.append(value) continue definition_identity = (value.definition_name, value.definition_version) if definition_identity not in manifests: manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) manifest = manifests[definition_identity] - if ( - manifest is None - or manifest.fingerprint != binding["manifest_fingerprint"] - or (identity in fingerprints and fingerprints[identity] != manifest.fingerprint) - ): + if manifest is None: + log_evidence_neutralized("no_manifest", unit="manifest") evidence.append(value) continue - try: - declaration = MemoryEvidenceDeclaration.model_validate_json( - stored_bytes(binding["declaration"], column="declaration") - ) - except (InvalidStoredColumnError, TypeError, ValueError): - declaration = MemoryEvidenceDeclaration() evidence.append( - value.model_copy( - update={ - "manifest_fingerprint": manifest.fingerprint, - "declaration": declaration, - } - ) + _apply_bound_declaration(value, binding, manifest, fingerprints.get(identity, manifest.fingerprint)) ) return tuple(evidence) @@ -712,25 +741,30 @@ async def _bind_memory_evidence( scope_id: str, ref: SourceRef, source: Source, + native_manifests: dict[tuple[str, str], SourceDefinitionManifest | None], ) -> None: """Freeze registration evidence only for a newly accepted Source row.""" if isinstance(source, SourceObservation): manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) if manifest is not None and manifest.fingerprint != source.definition_fingerprint: + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) manifest = None else: definition = self._registry.definition_for_source(source) - manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + identity = (definition.name, definition.version) + if identity not in native_manifests: + native_manifests[identity] = await SourceDefinitionManifestRepository().register_native( + connection, definition + ) + manifest = native_manifests[identity] if manifest is not None: # A native decoder can accept representations its generated schema # cannot describe. Such captures are valid, but cannot attest evidence. try: - Draft202012Validator.check_schema(manifest.source_schema) - Draft202012Validator(manifest.source_schema, registry=Registry()).validate( - source.model_dump(mode="json", by_alias=True) - ) + _source_validator(manifest).validate(source.model_dump(mode="json", by_alias=True)) except (JsonSchemaValidationError, SchemaError, Unresolvable): + log_evidence_neutralized("schema_mismatch", unit="source", level=logging.DEBUG) manifest = None declaration = ( MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) @@ -1008,6 +1042,32 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded +def _apply_bound_declaration( + value: SourceEvidence, + binding: Mapping[Any, Any], + manifest: SourceDefinitionManifest, + expected_fingerprint: str | None, +) -> SourceEvidence: + """Apply only a declaration bound to the verified historical manifest.""" + + if manifest.fingerprint != binding["manifest_fingerprint"] or expected_fingerprint != manifest.fingerprint: + log_evidence_neutralized("digest_mismatch", unit="manifest", level=logging.WARNING) + return value + try: + declaration = MemoryEvidenceDeclaration.model_validate_json( + stored_bytes(binding["declaration"], column="declaration") + ) + except (InvalidStoredColumnError, TypeError, ValueError): + log_evidence_neutralized("decode_failed", unit="declaration", level=logging.WARNING) + declaration = MemoryEvidenceDeclaration() + return value.model_copy( + update={ + "manifest_fingerprint": manifest.fingerprint, + "declaration": declaration, + } + ) + + def _source_material_digest(source: Source) -> str: # Preserve every material field except the narrow server witness upgrade. material = source.model_dump(mode="json", by_alias=True) diff --git a/src/powercontext/builtin/runtime/__init__.py b/src/powercontext/builtin/runtime/__init__.py index b4505c8128..92440feaec 100644 --- a/src/powercontext/builtin/runtime/__init__.py +++ b/src/powercontext/builtin/runtime/__init__.py @@ -42,6 +42,7 @@ GetDreamRunRequest, ListDreamRunsRequest, ) +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation from powercontext.builtin.review.generation import ( GeneratedCandidateResult, GenerationCapabilityUnavailableError, @@ -253,6 +254,7 @@ "ArtifactProcessingWorkerCompletion", "ArtifactProcessingWorkerFailure", "ArtifactProcessingWorkerOutcome", + "AtomicMemoryEvidenceAnnotation", "AtomicMemoryPage", "AtomicMemoryRuntimeApplication", "AtomicMemorySearchHit", diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index c2c030ecda..5ddd157403 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -3351,6 +3351,12 @@ async def state(self, artifact_id: str, *, context=None): async with self._runtime._scope_operation(self.scope_id): return await self._scoped.state(artifact_id, context=context) + async def evidence(self, artifact_id: str, *, revision: int, context=None): + """Read one authorized revision's neutral evidence facts.""" + + async with self._runtime._scope_operation(self.scope_id): + return await self._scoped.evidence(artifact_id, revision=revision, context=context) + async def _filter_current_candidates(self, candidates, *, context=None): async with self._runtime._scope_operation(self.scope_id): return await self._scoped._filter_current_candidates(candidates, context=context) diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 1b2ba69d4c..d0067d5b8c 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -19,6 +19,7 @@ import json from collections.abc import AsyncIterator, Callable from hashlib import sha256 +from itertools import groupby from time import perf_counter from pydantic import BaseModel, Field @@ -151,12 +152,16 @@ async def _evidence_history( ).all() if not rows: break - for row in rows: - scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) - reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) - artifact = await artifacts.get(connection, scope_id, reference) - yield await evidence.verify(connection, scope_id, reference, artifact.lineage.sources, sources=sources) - after = scope_id, artifact_id, revision + # Rows are ordered by scope, so one page holds contiguous per-scope groups. + for scope_id, group in groupby(rows, key=lambda row: str(row.scope_id)): + references = tuple( + ArtifactRef(family=AtomicMemory.family, artifact_id=str(row.artifact_id), revision=int(row.revision)) + for row in group + ) + loaded = await artifacts.get_many(connection, scope_id, references) + for reference, artifact in zip(references, loaded, strict=True): + yield await evidence.verify(connection, scope_id, reference, artifact.lineage.sources, sources=sources) + after = scope_id, reference.artifact_id, reference.revision def _obsolete_evidence(): @@ -402,12 +407,16 @@ async def rebuild_atomic_memory_current( index: AtomicMemoryIndex, *, batch_size: int = 100, + sources: SourceRepository | None = None, guard: Callable[[], None] = lambda: None, ) -> AtomicMemoryProjectionRebuildReport: """Repair registered no-vector rows inside the maintenance owner's transaction. The caller validates the existing schema, owns backup/recovery and commits only after full Server validation. No initialization or external I/O occurs. + ``sources`` is the deployment's Source registry: passing it keeps this repair's + evidence verdict identical to the runtime read path. A maintenance bundle has no + host registry, so it omits this and admission uses ``BUILTIN_SOURCE_REGISTRY``. """ if index.capabilities.embedding_profile is not None: raise AtomicMemoryIndexError("embedding-profile", "Registered repair requires a no-vector profile") @@ -425,9 +434,14 @@ async def rebuild_atomic_memory_current( await publisher.remove(connection, scope_id, artifact_id) after = scope_id, artifact_id await connection.execute(index.table.delete().where(_obsolete_current(index))) - await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, guard=guard) + await rebuild_atomic_memory_evidence(connection, index, batch_size=batch_size, sources=sources, guard=guard) return await verify_atomic_memory_current( - connection, index, batch_size=batch_size, require_evidence=True, require_evidence_projection=True + connection, + index, + batch_size=batch_size, + sources=sources, + require_evidence=True, + require_evidence_projection=True, ) diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index a80ff5df1f..0f24d550c6 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -929,6 +929,9 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( + # A host may replace only the evidence declaration while keeping the + # builtin adapter, version and projections. `memory_evidence` is + # deliberately excluded: it is the dimension this registration adds. not ( definition is expected or ( diff --git a/src/powercontext/errors.py b/src/powercontext/errors.py index 1ad41604c4..c45dd54889 100644 --- a/src/powercontext/errors.py +++ b/src/powercontext/errors.py @@ -25,6 +25,17 @@ class SourceError(PowerContextError): """Base exception for Source adapter and access failures.""" +class AtomicMemoryEvidenceError(PowerContextError): + """Raised when retained evidence no longer matches the exact accepted material. + + A caller can distinguish this integrity failure from retrieval or projection + errors through the stable :class:`PowerContextError` family. + """ + + def __init__(self) -> None: + super().__init__("Historical evidence does not match the exact Source material") + + class SourceNotFoundError(SourceError, LookupError): """Raised when a Source object is absent from a catalog.""" diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 8de06de50c..83306ee39d 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -581,6 +581,9 @@ async def _atomic_memory_maintenance( if action in {"apply", "rebuild-projection"}: embedding_model, _readiness = await _embedding_models(inference, resources, None) if action == "rebuild-projection": + # This command reads deployment configuration, not host code, so + # evidence admission uses BUILTIN_SOURCE_REGISTRY. Hosts with native + # Definitions pass the runtime registry to the Python API instead. report = await rebuild_atomic_memory_projection( profile.database, index, diff --git a/src/powercontext/server/logging.py b/src/powercontext/server/logging.py index 57aabaa25c..ea3eb94ccc 100644 --- a/src/powercontext/server/logging.py +++ b/src/powercontext/server/logging.py @@ -94,6 +94,7 @@ "verdict", "final_action", "fallback_reason", + "reason", "candidate_count", "evidence_count", "trace_id", @@ -197,7 +198,7 @@ def filter(self, record: logging.LogRecord) -> bool: if field not in self._identity and field not in {"request_id", "trace_id", "span_id"} ) if isinstance(event, str) and event.startswith("code."): - fields += _CODE_FIELDS + fields = tuple(dict.fromkeys((*fields, *_CODE_FIELDS))) record.processing_context = "".join( f" {field}={json.dumps(value, ensure_ascii=False)}" for field in fields diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index 18f6bd5162..0f3b9612f1 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -16,6 +16,7 @@ from __future__ import annotations +import logging from collections.abc import Iterable, Mapping from dataclasses import dataclass from types import MappingProxyType @@ -35,7 +36,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter -from powercontext.sources.evidence import MemoryEvidenceDeclaration +from powercontext.sources.evidence import MemoryEvidenceDeclaration, log_evidence_neutralized from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -105,6 +106,9 @@ class SourceDefinitionRegistry: manifest identities, never worker payload or projection values. Declarations are validated and copied at construction; invalid or unavailable values are neutral and never prevent registration of an otherwise valid Definition. + Only host code constructing this registry supplies ``remote_declarations``; + no configuration file entry exists, because the keys must match manifests + that the deployment itself registered. """ def __init__( @@ -257,6 +261,7 @@ def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: value = value.model_dump(mode="python") return MemoryEvidenceDeclaration.model_validate(value) except (TypeError, ValueError): + log_evidence_neutralized("invalid_declaration", unit="declaration", level=logging.WARNING) return _NEUTRAL_MEMORY_EVIDENCE diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py index 33eebfed9f..d2bcb99059 100644 --- a/src/powercontext/sources/evidence.py +++ b/src/powercontext/sources/evidence.py @@ -16,12 +16,57 @@ from __future__ import annotations +import logging from typing import Literal from pydantic import BaseModel, ConfigDict, Field +from powercontext._logging import log_safely from powercontext.sources.models import SourceMaterialization, SourceRef +_EVIDENCE_LOGGER = logging.getLogger(__name__) + +EvidenceNeutralReason = Literal[ + "schema_mismatch", + "digest_mismatch", + "no_manifest", + "decode_failed", + "legacy_unverifiable", + "invalid_declaration", + "manifest_conflict", + "definition_missing", + "evidence_mismatch", +] +"""Bounded reasons why accepted evidence resolved to neutral. + +The vocabulary is operational only: it names which retained fact could not be +attested, never the material, declaration body, or requested content. +""" + +EvidenceNeutralUnit = Literal["source", "declaration", "manifest", "revision"] + + +def log_evidence_neutralized( + reason: EvidenceNeutralReason, + *, + unit: EvidenceNeutralUnit, + level: int = logging.INFO, +) -> None: + """Emit one content-free record for a neutralized evidence fact.""" + + log_safely( + _EVIDENCE_LOGGER, + level, + "Memory evidence resolved to neutral", + extra={ + "event": "atomic_memory.evidence.neutral", + "operation": "atomic_memory.evidence", + "unit": unit, + "outcome": "neutral", + "reason": reason, + }, + ) + class MemoryEvidenceDeclaration(BaseModel): """A trusted adapter's claim, not a score, permission or verification engine. diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 843196a3c6..b27016dace 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,7 +17,8 @@ from __future__ import annotations import hashlib -from typing import Any, Literal +from types import UnionType +from typing import Any, Literal, Union, get_args, get_origin import rfc8785 from pydantic import ( @@ -248,30 +249,76 @@ def _validation_schema_matches_payload( source_class: type[BaseModel], seen: set[type[BaseModel]] | None = None, ) -> bool: - """Return whether validation-mode keys already describe the serialized payload.""" + """Return whether validation-mode keys accept the serialized payload.""" visited = set() if seen is None else seen if source_class in visited: return True visited.add(source_class) serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) + # Preserve existing permissive manifests; optional renamed keys only fail when extras are forbidden. + forbid_extra = source_class.model_config.get("extra") == "forbid" for name, field in source_class.model_fields.items(): if field.exclude: continue serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name - if field.is_required() and _validation_schema_key(name, field) != serialization_key: + if (field.is_required() or forbid_extra) and _validation_schema_key(name, field) != serialization_key: return False - annotation = field.annotation - if ( - field.is_required() - and isinstance(annotation, type) - and issubclass(annotation, BaseModel) - and not _validation_schema_matches_payload(annotation, visited) + if not _validation_schema_matches_annotation(field.annotation, visited): + return False + return True + + +def _validation_schema_matches_annotation(annotation: Any, seen: set[type[BaseModel]]) -> bool: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return _validation_schema_matches_payload(annotation, seen) + arguments = get_args(annotation) + if get_origin(annotation) in (Union, UnionType): + return _union_validation_covers_payloads(arguments, seen) + return all(_validation_schema_matches_annotation(argument, seen) for argument in arguments) + + +def _union_validation_covers_payloads(arguments: tuple[Any, ...], seen: set[type[BaseModel]]) -> bool: + # Every possible output needs coverage, not merely one compatible branch. + for branch in arguments: + if _validation_schema_matches_annotation(branch, set(seen)): + continue + if not isinstance(branch, type) or not issubclass(branch, BaseModel): + return False + serialized = branch.model_json_schema( + mode="serialization", by_alias=bool(branch.model_config.get("serialize_by_alias")) + ) + if not any( + isinstance(candidate, type) + and issubclass(candidate, BaseModel) + and _same_flat_schema(serialized, candidate.model_json_schema()) + for candidate in arguments ): return False return True +def _same_flat_schema(left: dict[str, Any], right: dict[str, Any]) -> bool: + """Prove flat model equivalence without interpreting references or nested schemas.""" + supported = {"title", "type", "properties", "required", "additionalProperties"} + if left.keys() - supported or right.keys() - supported: + return False + left_properties = left.get("properties", {}) + right_properties = right.get("properties", {}) + if left_properties.keys() != right_properties.keys(): + return False + for name, schema in left_properties.items(): + other = right_properties[name] + # Only primitive properties: no refs, unions, containers or nested schemas. + if schema.get("type") not in {"string", "integer", "number", "boolean", "null"}: + return False + if {key: value for key, value in schema.items() if key != "title"} != { + key: value for key, value in other.items() if key != "title" + }: + return False + return all(left.get(key) == right.get(key) for key in supported - {"title", "properties"}) + + def _validation_schema_key(name: str, field: Any) -> str | None: alias = field.validation_alias if isinstance(alias, str): diff --git a/tests/builtin/persistence/test_atomic_evidence_schema_copies.py b/tests/builtin/persistence/test_atomic_evidence_schema_copies.py new file mode 100644 index 0000000000..fe727cb88f --- /dev/null +++ b/tests/builtin/persistence/test_atomic_evidence_schema_copies.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keep the runtime evidence revision and the Server bundle on identical DDL.""" + +from __future__ import annotations + +import json +from pathlib import Path + +_PERSISTENCE = Path(__file__).resolve().parents[3] / "src/powercontext/builtin/persistence" +_RUNTIME = _PERSISTENCE / "migrations/resources/schemas/atomic_memory_evidence_v1.json" +_SERVER = _PERSISTENCE / "migrations/server_resources/schemas" + + +def test_runtime_and_server_evidence_ddl_are_identical() -> None: + # Runtime readiness compares SQLite `sqlite_master` text exactly, so a drift + # between these two producer copies fails managed readiness outside CI. + runtime = json.loads(_RUNTIME.read_text(encoding="utf-8")) + sqlite = json.loads((_SERVER / "sqlite-v1_5_0_001.json").read_text(encoding="utf-8")) + mysql = json.loads((_SERVER / "mysql-v1_5_0_001.json").read_text(encoding="utf-8")) + + assert runtime["sqlite"] == sqlite["create"] + assert runtime["mysql"] == mysql["revisions"]["v1_5_0_001"]["statements"] diff --git a/tests/builtin/persistence/test_evidence_neutralization_logging.py b/tests/builtin/persistence/test_evidence_neutralization_logging.py new file mode 100644 index 0000000000..d77175ccbd --- /dev/null +++ b/tests/builtin/persistence/test_evidence_neutralization_logging.py @@ -0,0 +1,166 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Neutral evidence facts report a bounded, content-free operational reason.""" + +from __future__ import annotations + +import asyncio +import json +import logging + +import pytest +from pydantic import field_serializer, field_validator +from sqlalchemy import update + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.server.logging import JsonFormatter, OperationalContextFilter +from powercontext.sources import ( + AdapterSourceDefinition, + MemoryEvidenceDeclaration, + SourceDefinitionRegistry, + SourceMaterialization, + project_source_for_transport, +) +from tests.builtin.persistence.contract import NoteAdapter, NoteSource, repository_profile + + +def _neutral_records(caplog) -> list[logging.LogRecord]: + return [record for record in caplog.records if getattr(record, "event", None) == "atomic_memory.evidence.neutral"] + + +def test_undecodable_snapshot_reports_decode_failed(caplog) -> None: + reference = ArtifactRef(family="atomic-memory", artifact_id="memory", revision=1) + + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + assert AtomicMemoryEvidenceRepository._decode("scope", reference, b"\xffnot-json") is None + + record = _neutral_records(caplog)[-1] + assert record.getMessage() == "Memory evidence resolved to neutral" + assert ( + getattr(record, "operation", None), + getattr(record, "unit", None), + getattr(record, "outcome", None), + getattr(record, "reason", None), + ) == ( + "atomic_memory.evidence", + "declaration", + "neutral", + "decode_failed", + ) + + +def test_neutral_reason_is_serialized_without_evidence_content(caplog) -> None: + reference = ArtifactRef(family="atomic-memory", artifact_id="memory", revision=1) + + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + AtomicMemoryEvidenceRepository._decode("scope", reference, b"\xffnot-json") + + record = _neutral_records(caplog)[-1] + OperationalContextFilter().filter(record) + payload = json.loads(JsonFormatter().format(record)) + + assert payload["event"] == "atomic_memory.evidence.neutral" + assert (payload["operation"], payload["unit"], payload["outcome"], payload["reason"]) == ( + "atomic_memory.evidence", + "declaration", + "neutral", + "decode_failed", + ) + assert set(payload) <= { + "timestamp", + "level", + "logger", + "message", + "event", + "operation", + "unit", + "outcome", + "reason", + "request_id", + "trace_id", + "span_id", + "service_name", + "service_version", + "service_role", + "service_instance_id", + } + + +def test_observation_fingerprint_mismatch_reports_digest_mismatch(caplog) -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry.from_adapters((NoteAdapter(),)) + source = NoteSource(name="observation", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + definition = registry.definition_for_name("note") + async with repository_profile() as (profile, _), profile.database.transaction() as connection: + await SourceDefinitionManifestRepository().register_native(connection, definition) + observation = project_source_for_transport(registry, source).model_copy( + update={"definition_fingerprint": "sha256:" + "0" * 64} + ) + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + await SourceRepository(registry).add(connection, "scope", observation) + records = _neutral_records(caplog) + assert [getattr(record, "reason", None) for record in records] == ["digest_mismatch"] + assert records[0].levelno == logging.WARNING + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("legacy_binding", [False, True]) +def test_schema_neutral_evidence_does_not_report_digest_corruption(caplog, legacy_binding: bool) -> None: + class SerializedNote(NoteSource): + @field_serializer("body") + def serialize_body(self, value: str): + return {"text": value} + + @field_validator("body", mode="before") + @classmethod + def deserialize_body(cls, value): + return value["text"] if isinstance(value, dict) else value + + class SerializedNoteAdapter(NoteAdapter): + source_class = SerializedNote + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + definition = AdapterSourceDefinition(SerializedNoteAdapter(), memory_evidence=declaration) + repository = SourceRepository(SourceDefinitionRegistry((definition,))) + source = SerializedNote( + name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained text." + ) + async with repository_profile() as (profile, _), profile.database.transaction() as connection: + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + stored = await repository.add(connection, "scope", source) + assert [getattr(record, "reason", None) for record in _neutral_records(caplog)] == ["schema_mismatch"] + if legacy_binding: + manifest = await SourceDefinitionManifestRepository().get(connection, "note", "1") + await connection.execute( + update(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + manifest_fingerprint=manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="powercontext.sources.evidence"): + evidence = await repository.evidence(connection, "scope", (stored.ref,)) + assert evidence[0].declaration == MemoryEvidenceDeclaration() + assert not any(record.levelno >= logging.WARNING for record in _neutral_records(caplog)) + if legacy_binding: + assert [getattr(record, "reason", None) for record in _neutral_records(caplog)] == ["schema_mismatch"] + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_mysql_initialization_preflight.py b/tests/builtin/persistence/test_mysql_initialization_preflight.py index 2c97dc4f78..c0e9ecd0d6 100644 --- a/tests/builtin/persistence/test_mysql_initialization_preflight.py +++ b/tests/builtin/persistence/test_mysql_initialization_preflight.py @@ -149,3 +149,127 @@ def execute_revision(connection, _config, _source, target, guard): assert completed.maintenance_window_id == "interrupted-initialization" finally: engine.dispose() + + +@pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) +@pytest.mark.parametrize("created_tables", [1, 2, 3]) +def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypatch, backend, created_tables): + from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner + from tests.builtin.persistence.test_atomic_evidence_upgrade_history import ( + configuration as evidence_configuration, + ) + from tests.builtin.persistence.test_atomic_evidence_upgrade_history import ( + previous_upstream_bundle, + ) + + configuration = evidence_configuration(None) + previous = previous_upstream_bundle(tmp_path / "previous-bundle", None) + database = tmp_path / "managed.db" + sqlite = SQLiteMigrationRunner(database, previous, configuration=configuration) + sqlite.apply(plan_id=sqlite.plan().plan_id, accepted=True, maintenance_confirmed=True) + directory = tmp_path / "native-evidence" + config = ( + SeekDBConfig(path=tmp_path / "native") + if backend == "seekdb" + else OceanBaseConfig(url=SecretStr("mysql+aoceanbase://root@localhost:2881/test?charset=utf8mb4")) + ) + connections = MaintenanceConnections( + config, evidence_directory=directory, lock_coordination="single-host" if backend == "oceanbase" else None + ) + runner = migration.MySQLMigrationRunner( + connections, server_bundle(), evidence_directory=directory, configuration=configuration + ) + identity = BackendIdentity(backend, "fixture-evidence-recovery", "test") + marker = "pc_atomic_memory_evidence_schema" + history = "pc_atomic_memory_evidence_history" + bindings = "pc_source_evidence_bindings" + inventory = copy.deepcopy( + next( + state + for state in runner.schema._prefixes["v1_5_0_001"] + if len({marker, history, bindings} & state["tables"].keys()) == created_tables + ) + ) + control = "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" + inventory["tables"]["pc_schema_revision"] = _parse_table(control)[1] + journal = migration._Journal(directory, identity) + journal.write( + migration._Evidence( + state="active", + database_id=identity.database_id, + source_revision=None, + source_fingerprint=runner.schema.fingerprint({"tables": {}, "views": [], "triggers": []}), + bundle_checksum=runner.checksum, + configuration_digest=runner.configuration_digest, + last_revision="v1_4_1_004", + pending_revision="v1_5_0_001", + maintenance_window_id="interrupted-evidence-ddl", + backup_policy="auto", + backup_state="not_required", + release_checksums={revision: runner.checksums[revision] for revision in previous.revisions}, + ) + ) + engine = create_engine(f"sqlite:///{database}") + try: + with engine.connect() as connection: + statements = json.loads((runner.bundle.snapshot_directory / "schemas/sqlite-v1_5_0_001.json").read_text())[ + "create" + ] + for statement in statements[:created_tables]: + connection.exec_driver_sql(statement) + connection.commit() + monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) + monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) + monkeypatch.setattr( + connections, "run", lambda operation, **_options: operation(connection, identity, lambda: None) + ) + monkeypatch.setattr(connections, "coordination", lambda _identity: {}) + before = database.read_bytes(), journal.path.read_bytes() + plan = runner.plan() + assert plan.state == "recovery_required" and plan.source_revision == "v1_4_1_004" + assert plan.revisions == ("v1_5_0_001", "v1_5_0_002") + assert (database.read_bytes(), journal.path.read_bytes()) == before + assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [] + + def execute_revision(connection, _config, _source, target, guard): + # Model the remaining native DDL, retaining the real domain work, + # version advancement, and original durable recovery evidence. + config = runner.bundle.config(connection) + config.attributes["configuration"] = runner.configuration + if target == "v1_5_0_001": + for statement in statements[created_tables:]: + connection.exec_driver_sql(statement) + command.stamp(config, target) + else: + command.upgrade(config, target) + runner.bundle.domains.apply( + connection, target, backend=backend, configuration=runner.configuration, guard=guard + ) + inventory.clear() + inventory.update(copy.deepcopy(runner.schema._schemas[target])) + inventory["tables"]["pc_schema_revision"] = _parse_table(control)[1] + connection.commit() + + monkeypatch.setattr(runner, "_upgrade", execute_revision) + result = runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert result.state == "ready" and result.revision == "v1_5_0_002" + assert runner.verify().state == "ready" + assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [ + ("p0005",) + ] + completed = journal.read() + assert completed is not None and completed.state == "complete" + assert completed.maintenance_window_id == "interrupted-evidence-ddl" + assert not runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True).changed + + # An empty marker is resumable only while the original evidence + # records pending DDL, never a repair authorization at completed head. + connection.exec_driver_sql("DELETE FROM pc_atomic_memory_evidence_schema") + connection.commit() + before = database.read_bytes(), journal.path.read_bytes() + for inspect in (runner.plan, runner.verify, runner.apply): + with pytest.raises(MigrationError, match="domain_validation_failed"): + inspect() + assert (database.read_bytes(), journal.path.read_bytes()) == before + finally: + engine.dispose() diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index d48af8f765..3b58c4184d 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -981,8 +981,17 @@ def load(connection: Connection | None, _identity: BackendIdentity, _guard: Call ("ALTER TABLE pc_artifact_candidate_versions RENAME TO pc_candidate_versions", "s1300"), ("UPDATE pc_artifact_processing_schema SET config_manifest", "s1300"), ("CREATE TABLE pc_memory_artifact_archive", "s1400"), + ("CREATE TABLE pc_atomic_memory_evidence_schema", "v1_4_1_004"), + ("CREATE TABLE pc_source_evidence_bindings", "v1_4_1_004"), ], - ids=("retained-copy", "candidate-rename", "domain-processing", "next-revision"), + ids=( + "retained-copy", + "candidate-rename", + "domain-processing", + "next-revision", + "evidence-marker", + "evidence-bindings", + ), ) def test_mysql_full_server_interruption_preserves_history_and_original_maintenance( backend: _BackendTarget, interruption: str, recorded_revision: str diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 66e9b3022b..8ddb5e49d2 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -16,8 +16,11 @@ import asyncio import json +import sqlite3 +from typing import cast from urllib.parse import quote, unquote +import aiosqlite import pytest from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel @@ -190,6 +193,92 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_evidence_reads_large_source_batches_with_sqlite_variable_limit() -> None: + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((AdapterSourceDefinition(SOURCE_ADAPTERS[1], memory_evidence=declaration),)) + repository = SourceRepository(registry) + sources = tuple( + NoteSource(name=f"note-{index}", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + for index in range(501) + ) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add_many_with_status(connection, "scope", sources) + refs = tuple(item.ref for item, _ in stored) + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + previous_limit = await driver._execute(driver._conn.setlimit, sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 999) + try: + evidence = await repository.evidence(connection, "scope", (*reversed(refs), refs[0])) + finally: + await driver._execute(driver._conn.setlimit, sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, previous_limit) + assert tuple(item.source_ref for item in evidence) == tuple(reversed(refs)) + assert all(item.declaration == declaration for item in evidence) + + asyncio.run(scenario()) + + +def test_source_batches_register_evidence_again_after_savepoint_rollback() -> None: + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + repository = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SOURCE_ADAPTERS[1], memory_evidence=declaration),)) + ) + sources = tuple( + NoteSource(name=f"note-{index}", materialization=SourceMaterialization.CAPTURED, body="Retained text.") + for index in range(2) + ) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + async with connection.begin_nested() as savepoint: + await repository.add_many_with_status(connection, "scope", sources) + await savepoint.rollback() + stored = await repository.add_many_with_status(connection, "scope", sources) + assert all(created for _, created in stored) + evidence = await repository.evidence(connection, "scope", tuple(item.ref for item, _ in stored)) + assert all(item.declaration == declaration for item in evidence) + + asyncio.run(scenario()) + + +def test_source_batch_validates_each_capture_against_reused_manifest() -> None: + class MixedSource(SerializationAliasSource): + body: str + + @field_serializer("body") + def serialize_body(self, value: str): + return {"text": value} if value == "incompatible" else value + + @field_validator("body", mode="before") + @classmethod + def deserialize_body(cls, value): + return value["text"] if isinstance(value, dict) else value + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(MixedSource), version="2", memory_evidence=declaration), + )) + sources = [ + await registry.resolve(SerializationAliasInput(source_id=f"note-{index}", body=body)) + for index, body in enumerate(("compatible", "incompatible", "compatible")) + ] + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add_many_with_status(connection, "scope", sources) + evidence = await repository.evidence(connection, "scope", tuple(item.ref for item, _ in stored)) + assert [item.declaration for item in evidence] == [declaration, MemoryEvidenceDeclaration(), declaration] + + asyncio.run(scenario()) + + def test_reusing_a_stable_source_identity_with_different_payload_is_a_conflict() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index dd60861c1a..3775e9c2aa 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -33,6 +33,7 @@ AtomicMemoryExtractionOutput, ) from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.persistence import InvalidStoredPayloadError from powercontext.builtin.persistence.tables import SOURCES_TABLE from powercontext.builtin.runtime import ( BuiltinConfig, @@ -41,6 +42,7 @@ open_builtin_runtime, ) from powercontext.builtin.runtime.atomic_memory_rebuild import ( + rebuild_atomic_memory_current, rebuild_atomic_memory_projection, verify_atomic_memory_current, ) @@ -443,3 +445,57 @@ async def scenario() -> None: assert await memory.evidence("a", revision=1) == accepted asyncio.run(scenario()) + + +def test_current_rebuild_accepts_the_configured_source_registry(migration_backend: MigrationBackend) -> None: + class AliasedNote(NoteSource): + # Retained material keys this identity by its business alias, so only the + # registered Definition can decode it; the builtin registry cannot. + name: str = Field(alias="source_name") + + class AliasedAdapter(NoteAdapter): + source_class = AliasedNote + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(AliasedAdapter()),)) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = AliasedNote.model_validate({ + "source_name": "aliased", + "materialization": "captured", + "body": "Retained.", + }) + await context.sources.add(source) + ref = SourceRef(source_type="note", source_id=source.name) + await _commit(contexts, "scope", "rebuild", "Retained fact.", sources=(ref,)) + # Replay a row captured before evidence bindings existed: no identity + # header, only the business-aliased native material. + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == "aliased").values(payload=payload) + ) + memory = contexts.atomic_memory.for_scope("scope") + expected = await memory.evidence("rebuild", revision=1) + + # Maintenance must judge the row exactly as the configured read path does; + # the builtin registry cannot decode it and must not report success. + with pytest.raises(InvalidStoredPayloadError): + async with contexts.database.transaction() as connection: + await rebuild_atomic_memory_current(connection, contexts.atomic_memory.index) + + async with contexts.database.transaction() as connection: + report = await rebuild_atomic_memory_current( + connection, contexts.atomic_memory.index, sources=contexts.repositories.sources + ) + assert report.ready, report.errors + assert await memory.evidence("rebuild", revision=1) == expected + + asyncio.run(scenario()) diff --git a/tests/test_server_logging.py b/tests/test_server_logging.py index 97c3a182a7..147123f629 100644 --- a/tests/test_server_logging.py +++ b/tests/test_server_logging.py @@ -482,6 +482,7 @@ def test_json_formatter_emits_stable_operational_fields() -> None: "verdict": "deny", "final_action": "memory_write_hold", "fallback_reason": "decision_model_fallback", + "reason": "manifest_conflict", "candidate_count": 1, "evidence_count": 2, "ignored": "not serialized", @@ -508,6 +509,7 @@ def test_json_formatter_emits_stable_operational_fields() -> None: assert payload["verdict"] == "deny" assert payload["final_action"] == "memory_write_hold" assert payload["fallback_reason"] == "decision_model_fallback" + assert payload["reason"] == "manifest_conflict" assert (payload["candidate_count"], payload["evidence_count"]) == (1, 2) assert "ignored" not in payload diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 12e80e8101..645355a9c8 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -225,6 +225,48 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_optional_alias_with_forbidden_extras_accepts_the_worker_payload() -> None: + class OptionalAliasSource(Source): + model_config = ConfigDict(extra="forbid") + body: str = Field(default="Default body.", alias="wire_text") + + class OptionalAliasAdapter: + name = "optional-alias" + input_class = str + source_class = OptionalAliasSource + + async def resolve(self, value: str, /) -> OptionalAliasSource: + return OptionalAliasSource.model_validate({ + "name": "optional", + "materialization": SourceMaterialization.CAPTURED, + "body": value, + }) + + async def read(self, source: OptionalAliasSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(OptionalAliasAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the optional value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "optional" + assert isinstance(stored, SourceObservation) + assert stored.payload["body"] == "Retain the optional value." + + asyncio.run(scenario()) + + def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: class VersionedSource(Source): model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) @@ -277,7 +319,67 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_nullable_nested_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: +@pytest.mark.parametrize("wrapper", ["default", "nullable", "list"]) +def test_populated_optional_nested_alias_accepts_the_worker_payload(wrapper: str) -> None: + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True) + label: str = Field(alias="wire_label") + + class DefaultNestedSource(Source): + detail: Detail = Field(default_factory=lambda: Detail.model_validate({"label": "Default label."})) + + class NullableNestedSource(Source): + detail: Detail | None = None + + class ListNestedSource(Source): + detail: list[Detail] = Field(default_factory=list) + + source_model = { + "default": DefaultNestedSource, + "nullable": NullableNestedSource, + "list": ListNestedSource, + }[wrapper] + + class NestedAliasAdapter: + name = "optional-nested-alias" + input_class = str + source_class = source_model + + async def resolve(self, value: str, /) -> Source: + detail = Detail.model_validate({"label": value}) + return source_model.model_validate({ + "name": "optional-nested", + "materialization": SourceMaterialization.CAPTURED, + "detail": [detail] if wrapper == "list" else detail, + }) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(NestedAliasAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the nested value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + expected_detail = {"label": "Retain the nested value."} + assert receipt.source_ref.source_id == "optional-nested" + assert isinstance(stored, SourceObservation) + assert stored.payload["detail"] == ([expected_detail] if wrapper == "list" else expected_detail) + + asyncio.run(scenario()) + + +def test_nullable_nested_alias_manifest_accepts_empty_payload(tmp_path) -> None: class Detail(BaseModel): label: str = Field(alias="wire_label") @@ -306,14 +408,8 @@ async def scenario() -> None: source = await registry.resolve("Retained.") manifest = manifest_for_definition(definition) observation = project_source_for_transport(registry, source) - baseline = { - "name": definition.name, - "version": definition.version, - "source_schema": NullableSource.model_json_schema(), - "projections": [], - } - assert manifest.source_schema == NullableSource.model_json_schema() - assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + # The declaration must also accept a populated Detail, even for an empty observation. + assert manifest.source_schema == NullableSource.model_json_schema(mode="serialization", by_alias=False) assert observation.payload["detail"] is None config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-nested.db'}") @@ -515,3 +611,65 @@ def reject_request(*_args: object, **_kwargs: object) -> None: fingerprint=f"sha256:{'0' * 64}", source_schema={"$dynamicRef": "https://example.invalid/schema"}, ) + + +def test_covered_union_retains_registered_manifest_and_accepts_both_branches() -> None: + class Aliased(BaseModel): + model_config = ConfigDict(populate_by_name=True, extra="forbid") + label: str = Field(validation_alias="wire_label") + + class Canonical(BaseModel): + model_config = ConfigDict(extra="forbid") + label: str + + class UnionSource(Source): + item: Aliased | Canonical + + class UnionAdapter: + name = "covered-union" + input_class = str + source_class = UnionSource + + async def resolve(self, value: str, /) -> UnionSource: + return UnionSource.model_validate({ + "name": value, + "definition_version": "1", + "materialization": "captured", + "item": {"label": value}, + }) + + async def read(self, source: UnionSource, /) -> str: + return source.item.label + + async def scenario() -> None: + definition = AdapterSourceDefinition(UnionAdapter(), version="1") + declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": UnionSource.model_json_schema(), + "projections": [], + } + historical = SourceDefinitionManifest.model_validate( + declaration + | { + "fingerprint": "sha256:" + hashlib.sha256(rfc8785.dumps(declaration)).hexdigest(), + } + ) + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(historical) + await contexts.register_source_definition(manifest_for_definition(definition)) + for index, branch in enumerate((Aliased.model_validate({"label": "a"}), Canonical(label="b"))): + source = UnionSource.model_validate({ + "name": f"branch-{index}", + "definition_version": "1", + "materialization": "captured", + "item": branch, + }) + observation = project_source_for_transport(SourceDefinitionRegistry((definition,)), source) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == source.name + + asyncio.run(scenario()) From 08a4386b86a3d93a67ab1bd06630c2477886fe6c Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Fri, 30 Oct 2026 11:25:26 +0800 Subject: [PATCH 24/26] ci(e2e): budget the six Server recovery scenarios The checkpoint group held a 600s deadline for four parametrized cases. The evidence interruption points added in 34c5276f make six, and one case reloads the baseline, upgrades the full Server chain, recovers it and verifies history, so the group needed roughly 610s and the last case was killed by the deadline. Raise the group to 900s and accept the thirteen native recovery and maintenance cases that the four junit reports now contain. --- .github/workflows/e2e-harness.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index fa86390ae9..7feb980227 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -218,12 +218,14 @@ jobs: # Each group keeps a hard deadline. The six independent recovery scenarios # must not consume a single shared deadline before client-exit recovery runs. + # One scenario reloads the baseline, upgrades the full Server chain, recovers + # it, and verifies history, so the group budget covers 6 x ~110s plus collection. - name: Recover committed Server DDL and domain checkpoints on real OceanBase without skips env: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + timeout --signal=TERM --kill-after=10s 900s uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-checkpoints.xml @@ -262,8 +264,8 @@ jobs: Path(".powercontext/recall-effort-ci/full-server-maintenance.xml"), ) cases = [case for report in reports for case in ElementTree.parse(report).iter("testcase")] - if len(cases) != 11 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All eleven native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 13 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All thirteen native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log From 8444f0845bb389782c54265136a104e9035193b7 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Fri, 30 Oct 2026 16:50:40 +0800 Subject: [PATCH 25/26] fix(persistence,sources): keep accepted manifests and MySQL-family evidence startup A Source field whose default is null can reach the payload as null, so a renamed nested shape below it never reaches the material. Traversing such a field for schema coverage replaced the manifest a deployment already accepted: its fingerprint changed and re-registration failed with SourceConflictError even though the payload still carried null. Only required fields, and models that forbid extras, now decide between the validation and serialization schemas. Populating a null-defaulted field with a renamed key requires the nested model to serialize by alias, which keeps the validation keys authoritative and the fingerprint stable. Cover the upgrade path: the nullable nested alias keeps the accepted manifest, its null payload registers across two sessions, a populated field-name payload is rejected, and a nested model serializing by alias carries populated values under that same manifest. Document the rule in both locales. The MySQL-family table fast path returned before the registered additive evidence migration, so an unmanaged OceanBase or seekDB database created the evidence tables without their p0005 marker and every runtime open failed readiness with "Atomic Memory evidence schema version is unsupported". Run the migration before the dialect branch and leave evidence tables to their registered DDL, as the SQLite path already does. The upgrade digest is recomputed over the final persistence inputs. --- docs/en/docs/workflows/sources.md | 10 +- docs/zh/docs/workflows/sources.md | 8 +- release/upgrade-support.json | 4 +- .../builtin/persistence/schema.py | 16 ++-- src/powercontext/sources/observations.py | 14 ++- .../test_atomic_description_vector_layout.py | 54 +++++++++-- .../test_full_server_migrations.py | 4 +- .../test_mysql_initialization_preflight.py | 22 ++++- .../test_mysql_vector_admission.py | 2 +- .../test_release140_server_migrations.py | 4 +- .../test_atomic_memory_description_upgrade.py | 4 +- tests/test_source_observations.py | 92 +++++++++++++++++-- 12 files changed, 186 insertions(+), 48 deletions(-) diff --git a/docs/en/docs/workflows/sources.md b/docs/en/docs/workflows/sources.md index 7c6eebc5f0..51090d4a5a 100644 --- a/docs/en/docs/workflows/sources.md +++ b/docs/en/docs/workflows/sources.md @@ -56,10 +56,14 @@ Remote Source observations configure the same declaration in trusted deployment name, version, and manifest fingerprint. A missing, invalid, or unavailable declaration resolves to `untrusted`/`unknown`; it does not block capture, processing, or existing retrieval. -Observation manifests validate the worker's serialized payload, including populated optional fields and nested models. -Validation aliases do not require workers to rename serialized keys, including when a model forbids extra fields. +Observation manifests describe the serialized payload for every field the payload must carry. Validation aliases do +not require workers to rename serialized keys, including when a model forbids extra fields. For unions, an alias-renamed branch may retain the validation manifest when another branch has an equivalent flat -primitive-property schema covering its serialized output. Unproven branch coverage uses the serialization schema. +primitive-property schema covering its serialized output. Unproven branch coverage uses the serialization schema, except +for a field whose default is `null`: it can reach the payload as `null`, so a renamed shape below it never replaces an +accepted manifest. Populating such a field with a renamed key requires the nested model to serialize by alias, which +keeps the validation keys authoritative. Changing what a registered Definition serializes requires a new Definition +version. Each Memory revision records the declaration and exact Source materialization accepted with it. That retained evidence is authoritative and versioned. The index projection beside it is derived: maintenance can discard and rebuild it diff --git a/docs/zh/docs/workflows/sources.md b/docs/zh/docs/workflows/sources.md index d209c67d66..f4aa54661b 100644 --- a/docs/zh/docs/workflows/sources.md +++ b/docs/zh/docs/workflows/sources.md @@ -53,10 +53,12 @@ definition = AdapterSourceDefinition( 远程 Source observation 在受信任的部署代码中配置同一声明，键为精确的 Definition 名称、版本与 manifest fingerprint。声明缺失、无效或不可用时解析为 `untrusted`/`unknown`，不阻塞采集、处理或既有检索。 -Observation manifest 校验 worker 序列化后的 payload，包括已填充的可选字段与嵌套模型。 -即使模型禁止额外字段，validation alias 也不要求 worker 重命名序列化后的键。 +Observation manifest 描述 payload 必须携带的每个字段的序列化形态。即使模型禁止额外字段，validation alias 也不要求 +worker 重命名序列化后的键。 对于 Union，若另一分支的等价扁平基础字段 schema 能覆盖 alias 分支的序列化输出，可保留 validation manifest。 -无法证明分支覆盖时，使用 serialization schema。 +无法证明分支覆盖时使用 serialization schema，但默认值为 `null` 的字段除外：该字段在 payload 里可以是 `null`， +其下的改名形态不得替换已接受的 manifest。要让这样的字段以改名后的键出现，嵌套模型必须按 alias 序列化，从而保持 +validation 键的权威性。改变已注册 Definition 的序列化表示需要新的 Definition 版本。 每个 Memory revision 记录当时接受的声明与精确 Source materialization。保留的证据是权威且带版本的；其旁的索引 投影是派生数据，维护可以丢弃它并从保留事实重建，且不查询当前 registry。声明不增加排序权重、不改变分数， diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 301b64238b..6a61d4b083 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "49eb07ee2fe537ffd03d56e1fb23d5af44edbe940b3d020d4b3ea5b4f1fa5f5b", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Revision v1_5_0_001 follows v1_4_1_005 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_002 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above." + "persistence_sha256": "470e7af6568005a676a82b34b3eaa5fa1511bb2227e0b1926e0412239bc4ca01", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Revision v1_5_0_001 follows v1_4_1_005 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_002 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index afc2fb8c2d..ecf745c689 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -23,16 +23,17 @@ from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.schema import CreateTable, sort_tables -from powercontext.builtin.persistence.bootstrap import mysql_error_code - from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.bootstrap import mysql_error_code from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: """Create selected tables only for unmanaged storage; managed schemas are read-only. - Evidence tables are applied only through their registered additive migration. + Evidence tables are applied only through their registered additive migration on + every dialect, so a MySQL-family database never gets the tables without the + registered marker row. """ names = await connection.run_sync(lambda sync: set(inspect(sync).get_table_names())) @@ -44,14 +45,15 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) "unsupported_configuration", "The versioned database does not cover the requested feature tables." ) return + evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): + await apply_atomic_memory_evidence_migration(connection) if connection.dialect.name == "mysql": - await connection.run_sync(lambda sync: _create_mysql_tables(sync, tables)) + selected = [table for table in tables if table.name not in evidence_names] + await connection.run_sync(lambda sync: _create_mysql_tables(sync, selected)) return groups: dict[MetaData, list[Table]] = {} - evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} - if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): - await apply_atomic_memory_evidence_migration(connection) for table in tables: if table.name in evidence_names: continue diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index b27016dace..e7a070eb13 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -261,9 +261,17 @@ def _validation_schema_matches_payload( for name, field in source_class.model_fields.items(): if field.exclude: continue - serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name - if (field.is_required() or forbid_extra) and _validation_schema_key(name, field) != serialization_key: - return False + if field.is_required() or forbid_extra: + serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name + if _validation_schema_key(name, field) != serialization_key: + return False + # A null default means this field can reach the payload as null, so a renamed + # shape below it need not appear and cannot justify replacing a manifest a + # deployment already accepted. Populating one instead requires the nested + # model to serialize by alias, which keeps the validation keys authoritative, + # or a new Definition version. + if field.default is None and not forbid_extra: + continue if not _validation_schema_matches_annotation(field.annotation, visited): return False return True diff --git a/tests/builtin/persistence/test_atomic_description_vector_layout.py b/tests/builtin/persistence/test_atomic_description_vector_layout.py index 6ff90fcf8e..04ae4255ad 100644 --- a/tests/builtin/persistence/test_atomic_description_vector_layout.py +++ b/tests/builtin/persistence/test_atomic_description_vector_layout.py @@ -18,6 +18,7 @@ import shutil import sqlite3 from contextlib import closing +from pathlib import Path import pytest @@ -42,7 +43,7 @@ def test_managed_sqlite_vector_schema_includes_registered_nullable_description(t database = tmp_path / "vector.db" runner = SQLiteMigrationRunner(database, bundle, configuration=configuration) result = runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) - assert result.revision == "v1_4_1_005" + assert result.revision == "v1_5_0_002" with closing(sqlite3.connect(database)) as connection: columns = {row[1]: row for row in connection.execute("PRAGMA table_info(pc_atomic_memory_current)")} assert columns["description"][2:4] == ("VARCHAR(200)", 0) @@ -54,9 +55,12 @@ def test_mysql_vector_admission_retains_description_and_exact_previous_revision( plain = server_bundle() vector = server_bundle(embedding_profile=profile) assert vector.vector_profile is not None + historical_revisions = plain.revisions[: plain.revisions.index("v1_4_1_005") + 1] inventory = {"tables": {}, "views": [], "triggers": []} for resource in json.loads((plain.directory / "manifest.json").read_text())["mysql_state_layouts"]: - for entry in json.loads((plain.directory / resource).read_text())["revisions"].values(): + for revision, entry in json.loads((plain.directory / resource).read_text())["revisions"].items(): + if revision not in historical_revisions: + continue for change in entry["changes"]: for name, table in change.items(): if table is None: @@ -99,14 +103,39 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm shutil.copytree(current.snapshot_directory, directory, ignore=shutil.ignore_patterns("__pycache__")) manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - resources = manifest["revision_resources"].pop("v1_4_1_005") - for resource in (*resources, "versions/v1_4_1_005_atomic_memory_description.py", "mysql/versions/v1_4_1_005.py"): + appended = current.revisions[current.revisions.index("v1_4_1_004") + 1 :] + removed_resources = {resource for revision in appended for resource in manifest["revision_resources"].pop(revision)} + removed_steps = [step for step in manifest["domain_steps"] if step["revision"] in appended] + manifest["domain_steps"] = [step for step in manifest["domain_steps"] if step["revision"] not in appended] + for step in removed_steps: + removed_resources.update((step["module"], *step.get("resources", ()))) + retained_resources = {resource for resources in manifest["revision_resources"].values() for resource in resources} + for step in manifest["domain_steps"]: + retained_resources.update((step["module"], *step.get("resources", ()))) + for name in appended: + revision = current.script.get_revision(name) + removed_resources.add(Path(revision.path).relative_to(current.snapshot_directory).as_posix()) + removed_resources.add(f"mysql/versions/{name}.py") + for resource in removed_resources - retained_resources: (directory / resource).unlink() - del manifest["sqlite_fingerprints"]["v1_4_1_005"] - del manifest["domain_readiness"]["v1_4_1_005"] - manifest["mysql_state_layouts"].remove("schemas/mysql-v1_4_1_005.json") + for revision in appended: + del manifest["sqlite_fingerprints"][revision] + del manifest["domain_readiness"][revision] + manifest.get("mysql_wrappers", {}).pop(revision, None) + manifest["mysql_state_layouts"] = [ + resource for resource in manifest["mysql_state_layouts"] if resource not in removed_resources + ] + evidence_tables = { + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence", + "pc_source_evidence_bindings", + } + manifest["affected_objects"] = [name for name in manifest["affected_objects"] if name not in evidence_tables] manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") previous = MigrationBundle(directory, embedding_profile=profile) + assert previous.head == "v1_4_1_004" + assert previous.checksums == {revision: current.checksums[revision] for revision in previous.revisions} configuration = { "embedding_enabled": profile is not None, "processing_manifest": canonical_processing_manifest(BuiltinConfig()), @@ -119,11 +148,16 @@ def test_description_upgrade_preserves_registered_owner_recovery_and_evidence(tm owner_record = ("original-operation", "original-deployment", "original-scope", 0, '{"principal":"原始身份"}') with closing(sqlite3.connect(database)) as connection, connection: connection.execute("INSERT INTO pc_access_owner_recovery VALUES (?,?,?,?,?)", owner_record) - original_evidence = json.loads(old.evidence_path.read_text(encoding="utf-8")) + original_evidence_bytes = old.evidence_path.read_bytes() + original_evidence = json.loads(original_evidence_bytes) upgraded = SQLiteMigrationRunner(database, current, configuration=configuration) plan = upgraded.plan() - assert plan.source_revision == "v1_4_1_004" and plan.revisions == ("v1_4_1_005",) - assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_4_1_005" + assert plan.source_revision == "v1_4_1_004" + assert plan.target_revision == "v1_5_0_002" + assert plan.revisions == ("v1_4_1_005", "v1_5_0_001", "v1_5_0_002") + assert plan.domain_steps == ("atomic_memory_evidence", "atomic_memory_evidence_projection") + assert upgraded.evidence_path.read_bytes() == original_evidence_bytes + assert upgraded.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).revision == "v1_5_0_002" assert upgraded.verify().state == "ready" evidence = json.loads(upgraded.evidence_path.read_text(encoding="utf-8")) for revision, checksum in original_evidence["release_checksums"].items(): diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index d2d5ee1177..5f922f5fa7 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -74,9 +74,7 @@ def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() assert plan.source_revision == "s1300" and plan.target_revision == "v1_5_0_002" - assert plan.revisions == ( - "s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_5_0_001", "v1_5_0_002" - ) + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_5_0_001", "v1_5_0_002") assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" diff --git a/tests/builtin/persistence/test_mysql_initialization_preflight.py b/tests/builtin/persistence/test_mysql_initialization_preflight.py index c0e9ecd0d6..6db74dc4aa 100644 --- a/tests/builtin/persistence/test_mysql_initialization_preflight.py +++ b/tests/builtin/persistence/test_mysql_initialization_preflight.py @@ -164,6 +164,8 @@ def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypa configuration = evidence_configuration(None) previous = previous_upstream_bundle(tmp_path / "previous-bundle", None) + assert previous.head == "v1_4_1_004" + predecessor = "v1_4_1_005" database = tmp_path / "managed.db" sqlite = SQLiteMigrationRunner(database, previous, configuration=configuration) sqlite.apply(plan_id=sqlite.plan().plan_id, accepted=True, maintenance_confirmed=True) @@ -201,23 +203,33 @@ def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypa source_fingerprint=runner.schema.fingerprint({"tables": {}, "views": [], "triggers": []}), bundle_checksum=runner.checksum, configuration_digest=runner.configuration_digest, - last_revision="v1_4_1_004", + last_revision=predecessor, pending_revision="v1_5_0_001", maintenance_window_id="interrupted-evidence-ddl", backup_policy="auto", backup_state="not_required", - release_checksums={revision: runner.checksums[revision] for revision in previous.revisions}, + release_checksums={ + revision: runner.checksums[revision] + for revision in runner.bundle.revisions[: runner.bundle.revisions.index(predecessor) + 1] + }, ) ) engine = create_engine(f"sqlite:///{database}") try: with engine.connect() as connection: + # The interrupted evidence DDL starts only after the released + # description migration, including its real version advancement. + config = runner.bundle.config(connection) + config.attributes["configuration"] = configuration + command.upgrade(config, predecessor) + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == predecessor statements = json.loads((runner.bundle.snapshot_directory / "schemas/sqlite-v1_5_0_001.json").read_text())[ "create" ] for statement in statements[:created_tables]: connection.exec_driver_sql(statement) connection.commit() + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == predecessor monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) monkeypatch.setattr( @@ -226,7 +238,7 @@ def test_evidence_recovery_resumes_before_final_schema_marker(tmp_path, monkeypa monkeypatch.setattr(connections, "coordination", lambda _identity: {}) before = database.read_bytes(), journal.path.read_bytes() plan = runner.plan() - assert plan.state == "recovery_required" and plan.source_revision == "v1_4_1_004" + assert plan.state == "recovery_required" and plan.source_revision == predecessor assert plan.revisions == ("v1_5_0_001", "v1_5_0_002") assert (database.read_bytes(), journal.path.read_bytes()) == before assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [] @@ -253,6 +265,7 @@ def execute_revision(connection, _config, _source, target, guard): monkeypatch.setattr(runner, "_upgrade", execute_revision) result = runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) assert result.state == "ready" and result.revision == "v1_5_0_002" + assert result.backup_state == "not_required" assert runner.verify().state == "ready" assert connection.exec_driver_sql("SELECT schema_version FROM pc_atomic_memory_evidence_schema").all() == [ ("p0005",) @@ -260,6 +273,9 @@ def execute_revision(connection, _config, _source, target, guard): completed = journal.read() assert completed is not None and completed.state == "complete" assert completed.maintenance_window_id == "interrupted-evidence-ddl" + assert completed.source_revision is None and completed.backup_state == "not_required" + assert completed.last_revision == "v1_5_0_002" and completed.pending_revision is None + assert completed.release_checksums == runner.checksums assert not runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True).changed # An empty marker is resumable only while the original evidence diff --git a/tests/builtin/persistence/test_mysql_vector_admission.py b/tests/builtin/persistence/test_mysql_vector_admission.py index 14a92a7721..d4651cdd6b 100644 --- a/tests/builtin/persistence/test_mysql_vector_admission.py +++ b/tests/builtin/persistence/test_mysql_vector_admission.py @@ -69,7 +69,7 @@ def runner(selected=bundle, configured=configuration): monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) - def run(operation, *, writable=False): + def run(operation, *, writable=False, allow_missing_database=False): assert not writable, "Admission must fail before requesting a write connection" engine = create_engine(f"sqlite:///{database}") try: diff --git a/tests/builtin/persistence/test_release140_server_migrations.py b/tests/builtin/persistence/test_release140_server_migrations.py index e7126e59d0..f1b3ccb7a8 100644 --- a/tests/builtin/persistence/test_release140_server_migrations.py +++ b/tests/builtin/persistence/test_release140_server_migrations.py @@ -134,9 +134,7 @@ def test_registered_sources_choose_distinct_lineages(tmp_path, release, expected _settings, _embedding, runner = _source(tmp_path, release=release) plan = runner.plan() assert plan.source_revision == expected - assert plan.revisions == ( - "s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_5_0_001", "v1_5_0_002" - ) + assert plan.revisions == ("s1400", "s1401", "s1402", "v1_4_1_004", "v1_4_1_005", "v1_5_0_001", "v1_5_0_002") @pytest.mark.parametrize(("vectors", "code"), [(False, False), (True, False), (False, True), (True, True)]) diff --git a/tests/e2e/test_atomic_memory_description_upgrade.py b/tests/e2e/test_atomic_memory_description_upgrade.py index 235d3aa032..01f656ee6e 100644 --- a/tests/e2e/test_atomic_memory_description_upgrade.py +++ b/tests/e2e/test_atomic_memory_description_upgrade.py @@ -85,9 +85,9 @@ def test_nonempty_released_memory_upgrades_without_rewriting_history(tmp_path, r ) result = _apply(runner) - assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_4_1_005" + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "v1_5_0_002" verified = runner.verify() - assert verified.state == "ready" and verified.readiness_scope == "server" + assert verified.state == "ready" and verified.readiness_scope == "server" and verified.revision == "v1_5_0_002" assert not _apply(runner).changed with closing(sqlite3.connect(runner.database)) as connection: assert ( diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 645355a9c8..be10272920 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -26,6 +26,7 @@ from pydantic.alias_generators import to_camel from referencing.exceptions import Unresolvable +from powercontext import InvalidSourceObservationError from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES @@ -319,8 +320,11 @@ async def scenario() -> None: asyncio.run(scenario()) -@pytest.mark.parametrize("wrapper", ["default", "nullable", "list"]) +@pytest.mark.parametrize("wrapper", ["default", "list"]) def test_populated_optional_nested_alias_accepts_the_worker_payload(wrapper: str) -> None: + # A field with a concrete default always reaches the payload, so a renamed nested + # shape below it must be described by the serialization schema. A null default is + # covered by test_nullable_nested_alias_upgrade_keeps_the_accepted_manifest. class Detail(BaseModel): model_config = ConfigDict(populate_by_name=True) label: str = Field(alias="wire_label") @@ -328,15 +332,11 @@ class Detail(BaseModel): class DefaultNestedSource(Source): detail: Detail = Field(default_factory=lambda: Detail.model_validate({"label": "Default label."})) - class NullableNestedSource(Source): - detail: Detail | None = None - class ListNestedSource(Source): detail: list[Detail] = Field(default_factory=list) source_model = { "default": DefaultNestedSource, - "nullable": NullableNestedSource, "list": ListNestedSource, }[wrapper] @@ -379,8 +379,11 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_nullable_nested_alias_manifest_accepts_empty_payload(tmp_path) -> None: +def test_nullable_nested_alias_upgrade_keeps_the_accepted_manifest(tmp_path) -> None: + """A payload that can only carry null cannot justify replacing an accepted manifest.""" + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True) label: str = Field(alias="wire_label") class NullableSource(Source): @@ -408,8 +411,15 @@ async def scenario() -> None: source = await registry.resolve("Retained.") manifest = manifest_for_definition(definition) observation = project_source_for_transport(registry, source) - # The declaration must also accept a populated Detail, even for an empty observation. - assert manifest.source_schema == NullableSource.model_json_schema(mode="serialization", by_alias=False) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": NullableSource.model_json_schema(), + "projections": [], + } + # Renaming the nested key only matters to material this Definition never emits. + assert manifest.source_schema == NullableSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" assert observation.payload["detail"] is None config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-nested.db'}") @@ -423,6 +433,72 @@ async def scenario() -> None: SubmitSourceObservation(scope_id="scope-a", observation=observation) ) assert receipt.source_ref.source_id == "nullable" + # Populating the field needs a representation the accepted manifest describes, + # so the flat field-name payload is rejected instead of silently accepted. + populated = project_source_for_transport( + registry, + NullableSource( + name="nullable", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + detail=Detail.model_validate({"label": "Populated."}), + ), + ) + with pytest.raises(InvalidSourceObservationError, match="schema"): + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=populated) + ) + + asyncio.run(scenario()) + + +def test_nullable_nested_alias_serializing_by_alias_accepts_populated_payloads(tmp_path) -> None: + """A nested model that serializes by alias keeps the accepted manifest and populated values.""" + + class Detail(BaseModel): + model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True) + label: str = Field(alias="wire_label") + + class NullableSource(Source): + detail: Detail | None = None + + class PopulatedAdapter: + name = "nullable-nested-alias" + input_class = str + source_class = NullableSource + + async def resolve(self, value: str, /) -> NullableSource: + return NullableSource( + name="populated", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + detail=Detail.model_validate({"label": value}), + ) + + async def read(self, source: NullableSource, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(PopulatedAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retain the nested value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + assert manifest.source_schema == NullableSource.model_json_schema() + assert observation.payload["detail"] == {"wire_label": "Retain the nested value."} + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "populated" + assert isinstance(stored, SourceObservation) + assert stored.payload["detail"] == {"wire_label": "Retain the nested value."} asyncio.run(scenario()) From c9cede7d9bad70719c05517152349080cf90541d Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 4 Nov 2026 09:29:21 +0800 Subject: [PATCH 26/26] fix(migrations): align evidence gates with upstream CI - Narrow the prepared context union before reading status/content so the repository type check accepts the reuse envelope. - Defer evidence DDL during the historical native conversion instead of creating it through the unmanaged bootstrap, which stamped p0005 early and made the registered marker INSERT fail as a duplicate. - Refresh the release review digest. --- release/upgrade-support.json | 2 +- .../builtin/persistence/migrations/historical.py | 12 ++++++++++++ .../test_atomic_memory_evidence_neutral_recall.py | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 591e3f2e13..4cc2af1241 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,7 +3,7 @@ "target_version": "1.5.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "c68f572a1a4e0de6a172fb70f0d1cf3a3c0c27ffbff67a0b4a0bf856c179f0ae", + "persistence_sha256": "1f44be0f7352b4f6dcd19ddd1f0e5ae10ad46121fb1eafc1cadb12869e21f27c", "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged." }, "cases": [], diff --git a/src/powercontext/builtin/persistence/migrations/historical.py b/src/powercontext/builtin/persistence/migrations/historical.py index 6c6d90bbb3..76ddb185af 100644 --- a/src/powercontext/builtin/persistence/migrations/historical.py +++ b/src/powercontext/builtin/persistence/migrations/historical.py @@ -32,6 +32,9 @@ from sqlalchemy.schema import CreateColumn from typing_extensions import override +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE + from .bundle import MigrationBundle from .models import MigrationError, digest from .pinned import finish, pinned @@ -464,6 +467,15 @@ async def initialize(self, connection, /): await self.rebuild_fts(connection) class HistoricalNativeAtomicIndex(OceanBaseAtomicMemoryIndex): + def __init__(self, profile): + super().__init__(profile) + # Frozen historical DDL predates the registered evidence revisions. + # Creating these tables here would apply runtime DDL outside those + # revisions, leaving the Server chain's marker INSERT unsatisfiable. + deferred = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + deferred.add(ATOMIC_MEMORY_EVIDENCE_PROJECTION_TABLE.name) + self.tables = tuple(table for table in self.tables if table.name not in deferred) + @override async def require_current_schema(self, connection): columns = await connection.run_sync(lambda c: inspect(c).get_columns("pc_atomic_memory_current")) diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py index c3ad6ee50b..b03dbb6924 100644 --- a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py +++ b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py @@ -29,6 +29,7 @@ BuiltinConfig, HandoffArtifactCitation, PrepareContextRequest, + PreparedContext, RuntimeConfig, open_builtin_contexts, open_builtin_runtime, @@ -112,6 +113,7 @@ async def scenario() -> None: tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled ) prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) + assert isinstance(prepared, PreparedContext) assert prepared.status == "ready" and prepared.content is not None items = json.loads(prepared.content.splitlines()[-2])["items"] assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled