From 4c147ae51b1aa713015d3161cf6b570407a292c2 Mon Sep 17 00:00:00 2001 From: lkx Date: Sat, 31 Oct 2026 15:34:34 +0800 Subject: [PATCH 1/2] test(seekdb): enforce native restart and recovery acceptance --- .github/ci-paths.toml | 2 +- .github/workflows/master.yml | 57 +++-- .github/workflows/seekdb-lock-observation.yml | 2 +- .github/workflows/seekdb-lock-probe.yml | 29 ++- docs/en/docs/operate/database-migrations.md | 41 ++++ docs/zh/docs/operate/database-migrations.md | 35 ++++ release/upgrade-support.json | 4 +- scripts/probe_seekdb_migration_lock.py | 194 +++++++++++++++--- .../test_mysql_migration_checks.py | 44 ++-- .../test_mysql_migration_runner.py | 49 ++--- .../test_mysql_vector_admission.py | 3 +- tests/builtin/test_code_publication.py | 6 +- tests/e2e/test_seekdb_restart_recovery.py | 193 +++++++++++++++++ .../test_topic_memory_recovery_migration.py | 11 +- tests/test_ops.py | 1 + tests/test_seekdb_migration_probe.py | 39 ++++ tox.ini | 9 +- 17 files changed, 587 insertions(+), 132 deletions(-) create mode 100644 tests/e2e/test_seekdb_restart_recovery.py create mode 100644 tests/test_seekdb_migration_probe.py diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 64753f7b4a..f3bdb066be 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -31,7 +31,7 @@ database = [ "tests/builtin/persistence/**", "tests/builtin/*native_code*", "tests/builtin/runtime/*recall*", "tests/builtin/runtime/test_statistics.py", "tests/builtin/runtime/test_model_usage_recorder.py", "tests/builtin/runtime/test_prepared_context.py", "tests/e2e/*atomic*", "tests/e2e/*mysql*", - "tests/e2e/*native_code*", "tests/e2e/test_recall_sufficiency_gate.py", + "tests/e2e/*native_code*", "tests/e2e/test_seekdb_restart_recovery.py", "tests/e2e/test_recall_sufficiency_gate.py", "tests/e2e/test_context_text_assembly.py", "tests/e2e/test_scope_search_migration.py", "scripts/run_atomic_memory_clean_migration.py", "scripts/check_atomic_memory_migration_shards.py", diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a661ba42aa..5626284dc2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -429,6 +429,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.14"] + migration-shard: [0, 1] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -442,9 +443,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Test code indexing with embedded seekdb + if: matrix.migration-shard == 0 run: make code-seekdb-test - name: Test database bootstrap with embedded seekdb without skips + if: matrix.migration-shard == 0 # Two 360s native-process deadlines still leave room for the remaining # bootstrap cases and diagnostic upload before the step deadline. timeout-minutes: 20 @@ -470,6 +473,7 @@ jobs: PYCODE - name: Test batched archive validation with embedded seekdb + if: matrix.migration-shard == 0 timeout-minutes: 5 run: | uv run --locked --extra seekdb python -c "import pylibseekdb" @@ -478,10 +482,11 @@ jobs: - name: Test unified migrations with embedded seekdb if: ${{ !cancelled() && steps.seekdb_setup.outcome == 'success' }} - # Full-Server cases run in separate processes and exceed fifteen minutes - # in aggregate; each runner keeps its 180-second deadline below. + # Last-client shutdown adds native startup work. Partition runner nodes; + # every node keeps its 180-second deadline. timeout-minutes: 25 env: + SEEKDB_MIGRATION_SHARD: ${{ matrix.migration-shard }} POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" PYTHONUNBUFFERED: "1" @@ -501,16 +506,18 @@ jobs: } trap collect_diagnostics EXIT df -h . /tmp - uv run --locked --extra seekdb pytest -vv \ - -o faulthandler_timeout=120 --durations=20 \ - tests/builtin/persistence/test_database_migration_backends.py \ - tests/builtin/persistence/test_database_backup_providers.py \ - tests/builtin/persistence/test_database_migration_connections.py \ - tests/e2e/test_topic_memory_recovery_migration.py \ - tests/builtin/persistence/test_mysql_migration_checks.py \ - tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ - 2>&1 | tee seekdb-migrations.log - uv run --locked --extra seekdb pytest --collect-only -q --color=no \ + if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then + uv run --locked --extra seekdb --extra code pytest -vv \ + -o faulthandler_timeout=120 --durations=20 \ + tests/builtin/persistence/test_database_migration_backends.py \ + tests/builtin/persistence/test_database_backup_providers.py \ + tests/builtin/persistence/test_database_migration_connections.py \ + tests/e2e/test_topic_memory_recovery_migration.py \ + tests/builtin/persistence/test_mysql_migration_checks.py \ + tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ + 2>&1 | tee seekdb-migrations.log + fi + uv run --locked --extra seekdb --extra code pytest --collect-only -q --color=no \ tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ 2>&1 | tee seekdb-runner-collection.log | tee -a seekdb-migrations.log mapfile -t seekdb_runner_nodes < <( @@ -521,25 +528,31 @@ jobs: echo "No seekdb migration runner tests were collected" >&2 exit 1 fi - for seekdb_runner_node in "${seekdb_runner_nodes[@]}"; do - uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \ + for index in "${!seekdb_runner_nodes[@]}"; do + if [ "$((index % 2))" -ne "$SEEKDB_MIGRATION_SHARD" ]; then + continue + fi + seekdb_runner_node="${seekdb_runner_nodes[$index]}" + uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \ "$seekdb_runner_node" \ 2>&1 | tee -a seekdb-migrations.log done # These are required independent runs, not retries: any failure stops # the job. Exercise the lifecycle race on both supported matrix ends. - for repetition in 2 3; do - echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log - uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \ - 'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \ - 2>&1 | tee -a seekdb-migrations.log - done + if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then + for repetition in 2 3; do + echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log + uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \ + 'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \ + 2>&1 | tee -a seekdb-migrations.log + done + fi - name: Upload seekdb bootstrap diagnostics if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-bootstrap-${{ matrix.python-version }} + name: seekdb-bootstrap-${{ matrix.python-version }}-${{ matrix.migration-shard }} path: .powercontext/database-bootstrap-ci/ if-no-files-found: ignore @@ -547,7 +560,7 @@ jobs: if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-migrations-${{ matrix.python-version }} + name: seekdb-migrations-${{ matrix.python-version }}-${{ matrix.migration-shard }} path: | seekdb-migrations.log seekdb-native.log diff --git a/.github/workflows/seekdb-lock-observation.yml b/.github/workflows/seekdb-lock-observation.yml index fc281623cf..43a8f889ec 100644 --- a/.github/workflows/seekdb-lock-observation.yml +++ b/.github/workflows/seekdb-lock-observation.yml @@ -36,7 +36,7 @@ jobs: # experiment records snapshots every 45 seconds for up to 20 minutes # per attempt, including beyond the observed 1000-second SQL timeout. uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ - --attempts 12 --timeout 1200 --output .powercontext/seekdb-lock-observation + --restart --attempts 12 --timeout 1200 --output .powercontext/seekdb-lock-observation - name: Upload observation evidence if: always() diff --git a/.github/workflows/seekdb-lock-probe.yml b/.github/workflows/seekdb-lock-probe.yml index 5172aa0c78..ad984f873e 100644 --- a/.github/workflows/seekdb-lock-probe.yml +++ b/.github/workflows/seekdb-lock-probe.yml @@ -5,6 +5,12 @@ on: paths: - scripts/probe_seekdb_migration_lock.py - .github/workflows/seekdb-lock-probe.yml + - tests/builtin/persistence/test_mysql_migration_runner.py + - tests/e2e/test_seekdb_restart_recovery.py + - tests/test_seekdb_migration_probe.py + - src/powercontext/builtin/persistence/** + - pyproject.toml + - uv.lock workflow_dispatch: permissions: @@ -17,22 +23,35 @@ concurrency: jobs: reproduce: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.14'] + scenario: [migration, restart-recovery] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - uses: ./.github/actions/setup-python-env with: - python-version: '3.11' + python-version: ${{ matrix.python }} - name: Reproduce migration lifecycle with complete native evidence - run: uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py --attempts 12 + run: | + uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario '${{ matrix.scenario }}' --restart --attempts 12 --timeout 180 - name: Upload native logs, lifecycle and live transaction snapshots if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-lock-probe-${{ github.sha }} - path: .powercontext/seekdb-lock-probe + name: seekdb-lock-probe-${{ matrix.python }}-${{ matrix.scenario }}-${{ github.sha }} + path: | + .powercontext/seekdb-lock-probe/environment.json + .powercontext/seekdb-lock-probe/attempt-*/*.json + .powercontext/seekdb-lock-probe/attempt-*/*.jsonl + .powercontext/seekdb-lock-probe/attempt-*/*.xml + .powercontext/seekdb-lock-probe/attempt-*/*.log + .powercontext/seekdb-lock-probe/attempt-*/*.gz include-hidden-files: true if-no-files-found: error retention-days: 7 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 95e76d08de..f9de1fbda8 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -435,6 +435,47 @@ Use the latest plan's `plan_id` and `coordination.pending_run_id`. Reconfirm the the original recovery point with a backup of partial state. Normal upgrades still use one confirmation. Recovery requires the additional explicit DBA declaration; PC never invents it when showing the plan. +## Native SeekDB restart and recovery acceptance + +The embedded engine must use `pylibseekdb>=1.4.0.post2,<2`. The native transaction recovery fix is supplied by +that dependency. The migration checks retain `SHOW TRIGGERS` and release the last client between operations. +Run the following from the repository root, separately under Python 3.11 and Python 3.14: + +```bash +uv sync --locked --extra seekdb +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario migration --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-migration +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart +``` + +Use a new output directory for each run. The migration scenario selects +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]`. The restart scenario checks metadata +inspection, commit, last-client shutdown, baseline DDL and preserved content and tags after reopening. Each attempt +must report exactly one passing test, with no skip, error or timeout. `--restart` also requires multiple distinct native +process IDs for the same directory. Evidence includes the Python and binding versions, native binary SHA-256, instance +lifetimes, SQL trace, compressed native logs and live transaction snapshots for slow attempts. The separate extended +observation workflow is diagnostic; its longer deadline does not replace the ordinary acceptance deadline. + +The frozen-fixture gate runs on Linux and observes kernel process-exit notifications before each reopen; +it does not use a fixed startup delay. For recovery acceptance, first reproduce the stall with an isolated `1.4.0.post1` environment and +`--scenario restart-recovery --keep-failed-storage`. Stop its engine before freezing the failed `tmp/pc-*/seekdb` +directory. Keep the original immutable. This scenario creates synthetic data in the `powercontext` database, including +the `preserved` content and `Keep` tag expected by the fixture test. A healthy database or a fixture with different +schema or sentinels cannot establish recovery of this defect. Compare the old and fixed engines on separate copies of +the same frozen fault fixture: + +```bash +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \ + --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture +``` + +The fixture gate opens only a disposable copy, creates and commits the baseline revision, verifies the original data, +reopens to verify it and commit a new write, then reopens once more to read that write. Preserve the fixture and the +old-engine failure evidence locally; CI uploads diagnostic files, not physical databases. Do not infer upgrade support +from these gates: the full installed-release upgrade matrix remains required. + ## Backup choices | Policy | Behavior | Options | diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index bf6b9ead5f..34bbd466c4 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -337,6 +337,41 @@ powercontext server db-migrate verify --env-file oceanbase.env \ 使用最新计划中的 `plan_id` 和 `coordination.pending_run_id`。重新声明既有手动备份，不用部分状态的新备份替换原恢复点。正常升级仍只确认一次；中断恢复另需用户显式作出 DBA 声明，PC 展示计划时不会自动替用户确认。 +## 原生 SeekDB 重启与恢复验收 + +嵌入式引擎必须使用 `pylibseekdb>=1.4.0.post2,<2`，事务恢复修复由该原生依赖提供。迁移检查保留 +`SHOW TRIGGERS`，并在操作之间释放最后一个客户端。分别在 Python 3.11 和 Python 3.14 环境中，从仓库根目录执行： + +```bash +uv sync --locked --extra seekdb +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario migration --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-migration +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart +``` + +每次运行使用新的输出目录。migration 场景选择原 issue 的 +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` 节点；restart-recovery 场景检查元数据查询、 +提交、最后一个客户端关闭、基线 DDL，以及重开后内容和标签的保留。每次必须恰好一个测试通过，跳过、报错和超时均不算通过。 +`--restart` 还要求同一目录经历多个不同的原生进程 PID。证据包括 Python 和绑定版本、原生二进制 SHA-256、实例生命周期、 +SQL 日志、压缩后的原生日志，以及慢执行时的事务快照。独立的延长观察工作流仅用于诊断，不能用它的较长时限替代正常验收时限。 + +故障库门禁在 Linux 上执行，每次重开前通过内核退出事件确认上一代进程已结束，不使用固定启动等待。 +故障库验收应先在隔离的 `1.4.0.post1` 环境中，使用 `--scenario restart-recovery --keep-failed-storage` 复现阻塞。 +停止引擎后，冻结失败输出中的 `tmp/pc-*/seekdb` 目录，并保持原件不变。此场景在 `powercontext` 数据库中创建合成数据， +包括故障库测试要求的 `preserved` 内容和 `Keep` 标签。健康库、结构或哨兵数据不同的库，不能用于证明该缺陷已恢复。 +使用同一冻结故障库的独立副本，对比旧引擎与修复后的引擎： + +```bash +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \ + --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture +``` + +故障库门禁只打开临时副本：创建并提交基线版本表、验证原始数据；重开后再次验证并提交新写入；第三次打开时读回该写入。 +故障库原件与旧版失败证据保存在本地；CI 只上传诊断文件，不上传物理数据库。这些门禁不代表完整升级支持， +发布仍须通过安装产物的全部升级矩阵。 + ## 备份选择 | 策略 | 行为 | 参数 | diff --git a/release/upgrade-support.json b/release/upgrade-support.json index f29f0f4319..faa4a1e2e7 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "6479e5cdb3ffeba35a45b1e1d0632b7890cbc2990c593b437c2e33b66116149b", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged." + "persistence_sha256": "e3a3265bf8e901025a58ad794f789f07e118fa3eac9c5573d2b2551eaa01b9c4", + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Code generation publication now names its existing 32-character identifier before the graph commit and records it in a derived publication.json cache sidecar. Existing generation manifests, graph row encodings, selected database identities and registered SQL layouts remain unchanged. Caches without publication records remain readable. Recovery retains identifiers reachable from current, previous or retained generation directories and scopes reclamation by binding. SQLite reclamation remains directory-owned. This review covers the code cache lifecycle changes since the preceding persistence review; it does not establish installed-release upgrade acceptance or change the upgrade-required publication policy." }, "cases": [], "automatic_sources": { diff --git a/scripts/probe_seekdb_migration_lock.py b/scripts/probe_seekdb_migration_lock.py index 9839491bce..cf403b3730 100644 --- a/scripts/probe_seekdb_migration_lock.py +++ b/scripts/probe_seekdb_migration_lock.py @@ -18,14 +18,18 @@ import argparse import gzip +import hashlib import json import os import platform import shutil +import signal import subprocess import sys import time +from contextlib import suppress from datetime import UTC, datetime +from importlib import import_module from importlib.metadata import version from pathlib import Path from types import ModuleType @@ -36,13 +40,25 @@ NODE = ( "tests/builtin/persistence/test_mysql_migration_runner.py::" - "test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]" + "test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]" ) +SCENARIOS = { + "migration": NODE, + "restart-recovery": ( + "tests/e2e/test_seekdb_restart_recovery.py::test_metadata_inspection_survives_last_client_restarts" + ), + "fixture-recovery": ( + "tests/e2e/test_seekdb_restart_recovery.py::test_saved_fault_recovers_and_remains_usable_after_reopen" + ), +} + def record(directory: Path, event: str, **values: object) -> None: with (directory / "lifecycle.jsonl").open("a") as output: - output.write(json.dumps({"time": datetime.now(UTC).isoformat(), "event": event, **values}) + "\n") + output.write( + json.dumps({"time": datetime.now(UTC).isoformat(), "event": event, **values}, ensure_ascii=False) + "\n" + ) def pytest_sessionstart(session: pytest.Session) -> None: @@ -53,29 +69,35 @@ def pytest_sessionstart(session: pytest.Session) -> None: original = profile._open_instance class ObservedInstance: - def __init__(self, instance: Any, path: Path) -> None: + def __init__(self, instance: Any, path: Path, native_pid: int) -> None: self.instance = instance self.path = path + self.native_pid = native_pid def connection_options(self) -> Any: return self.instance.connection_options() def close(self) -> None: - record(directory, "closing", path=str(self.path)) + record(directory, "closing", path=str(self.path), native_pid=self.native_pid) try: self.instance.close() finally: - record(directory, "close_returned", path=str(self.path)) + record(directory, "close_returned", path=str(self.path), native_pid=self.native_pid) async def observed_open(module: ModuleType, path: Path) -> profile._SeekDBInstance: record(directory, "opening", path=str(path)) instance = await original(module, path) + try: + native_pid = int((path / "run/seekdb.pid").read_text().strip()) + except (OSError, ValueError): + instance.close() + raise options = dict(instance.connection_options()) # Embedded scratch instance only. Never capture credentials or external connection settings. socket = options.get("unix_socket") (directory / "socket.json").write_text(json.dumps({"unix_socket": socket})) - record(directory, "opened", path=str(path)) - return ObservedInstance(instance, path) + record(directory, "opened", path=str(path), native_pid=native_pid) + return ObservedInstance(instance, path, native_pid) patch = pytest.MonkeyPatch() patch.setattr(profile, "_open_instance", observed_open) @@ -171,7 +193,42 @@ def successful_report(report: Path, code: int) -> bool: ) -def run_attempt(directory: Path, timeout: int) -> bool: +def stop_process(process: subprocess.Popen[bytes]) -> None: + """Bound shutdown of a probe child, including an unresponsive native client.""" + if process.poll() is None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def restart_evidence(directory: Path) -> dict[str, Any]: + """Require repeated native starts without reusing a process for the same target.""" + lifecycle = directory / "lifecycle.jsonl" + events = [json.loads(line) for line in lifecycle.read_text().splitlines()] if lifecycle.exists() else [] + processes: dict[str, list[int]] = {} + for event in events: + if event["event"] == "opened": + processes.setdefault(event["path"], []).append(event["native_pid"]) + passed = any(len(pids) > 1 for pids in processes.values()) and all( + all(pid > 0 for pid in pids) and len(set(pids)) == len(pids) for pids in processes.values() + ) + return {"passed": passed, "processes": processes} + + +def run_attempt( + directory: Path, + timeout: int, + *, + node: str = NODE, + restart: bool = False, + fixture: Path | None = None, + keep_failed_storage: bool = False, +) -> bool: directory.mkdir(parents=True) temporary = directory / "tmp" temporary.mkdir() @@ -181,6 +238,9 @@ def run_attempt(directory: Path, timeout: int) -> bool: POWERCONTEXT_SEEKDB_PROBE=str(directory), POWERCONTEXT_TEST_MIGRATION_SEEKDB="1", POWERCONTEXT_TEST_MIGRATION_SQL_LOG="1", + POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY="1", + POWERCONTEXT_SEEKDB_RECOVERY_FIXTURE=str(fixture or ""), + POWERCONTEXT_SEEKDB_KEEP_FAILED_STORAGE=str(int(keep_failed_storage)), ) report = directory / "junit.xml" command = [ @@ -194,9 +254,9 @@ def run_attempt(directory: Path, timeout: int) -> bool: "-vv", "--log-cli-level=INFO", "-o", - "faulthandler_timeout=120", + f"faulthandler_timeout={min(120, max(1, timeout // 2))}", f"--junitxml={report}", - NODE, + node, ] logs = NativeLogs(temporary, directory) snapshots: list[tuple[subprocess.Popen[bytes], Any, float]] = [] @@ -204,16 +264,24 @@ def run_attempt(directory: Path, timeout: int) -> bool: next_snapshot = 45.0 timed_out = False process = None + code = -1 + passed = False + restarts = None try: with (directory / "pytest.log").open("w") as output: - process = subprocess.Popen(command, env=environment, stdout=output, stderr=subprocess.STDOUT) # noqa: S603 + process = subprocess.Popen( # noqa: S603 + command, env=environment, stdout=output, stderr=subprocess.STDOUT, start_new_session=True + ) while process.poll() is None: logs.poll() elapsed = time.monotonic() - started if elapsed >= next_snapshot: log = (directory / f"snapshot-{int(next_snapshot)}.jsonl").open("w") probe = subprocess.Popen( # noqa: S603 - [sys.executable, __file__, "--snapshot", str(directory)], stdout=log, stderr=subprocess.STDOUT + [sys.executable, __file__, "--snapshot", str(directory)], + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, ) snapshots.append((probe, log, time.monotonic())) next_snapshot += 45 @@ -222,60 +290,120 @@ def run_attempt(directory: Path, timeout: int) -> bool: probe.kill() if elapsed >= timeout: timed_out = True - process.terminate() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() + stop_process(process) break time.sleep(0.1) code = process.wait() + passed = not timed_out and successful_report(report, code) + if restart and passed: + restarts = restart_evidence(directory) + passed = restarts["passed"] finally: - if process is not None and process.poll() is None: - process.kill() - process.wait() + if process is not None: + stop_process(process) for probe, output, _ in snapshots: - if probe.poll() is None: - probe.kill() - probe.wait() + stop_process(probe) output.close() logs.close() # Only remove this attempt's disposable engine files, after preserving native logs. - shutil.rmtree(temporary) - passed = not timed_out and successful_report(report, code) - result = {"passed": passed, "exit_code": code, "timed_out": timed_out, "seconds": time.monotonic() - started} + if not keep_failed_storage or passed: + shutil.rmtree(temporary) + result = { + "passed": passed, + "exit_code": code, + "timed_out": timed_out, + "seconds": time.monotonic() - started, + "restarts": restarts, + } (directory / "result.json").write_text(json.dumps(result, indent=2) + "\n") print(directory.name, json.dumps(result), flush=True) return passed -def main() -> int: +def positive_integer(value: str) -> int: + parsed = int(value) + if parsed < 1: + message = "must be positive" + raise argparse.ArgumentTypeError(message) + return parsed + + +def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--snapshot", type=Path) parser.add_argument("--output", type=Path, default=Path(".powercontext/seekdb-lock-probe")) - parser.add_argument("--attempts", type=int, default=12) - parser.add_argument("--timeout", type=int, default=180) + parser.add_argument("--attempts", type=positive_integer, default=12) + parser.add_argument("--node", help="Exact pytest node, overriding the selected scenario.") + parser.add_argument("--scenario", choices=SCENARIOS, default="migration") + parser.add_argument( + "--fixture", + type=Path, + help="Frozen synthetic baseline with the recovery test's sentinels; copied before opening.", + ) + parser.add_argument( + "--keep-failed-storage", action="store_true", help="Retain failed scratch storage for recovery acceptance." + ) + parser.add_argument("--restart", action="store_true", help="Require distinct native process IDs across reopenings.") + parser.add_argument("--timeout", type=positive_integer, default=180) arguments = parser.parse_args() + if arguments.snapshot: + return arguments + if arguments.scenario == "fixture-recovery" and arguments.fixture is None: + parser.error("--scenario fixture-recovery requires --fixture") + if arguments.fixture is not None: + arguments.fixture = arguments.fixture.resolve() + if not (arguments.fixture / "etc/seekdb.data_version.bin").is_file(): + parser.error("--fixture must be a recognized native seekdb directory") + node = arguments.node or SCENARIOS[arguments.scenario] + directory = arguments.output.resolve() + if arguments.fixture is not None and directory.is_relative_to(arguments.fixture): + parser.error("--output must be outside the immutable --fixture directory") + if directory.exists(): + parser.error("--output must be a new directory to preserve previous evidence") + arguments.output = directory + arguments.node = node + return arguments + + +def main() -> int: + arguments = parse_arguments() if arguments.snapshot: snapshot(arguments.snapshot) return 0 - directory = arguments.output.resolve() - directory.mkdir(parents=True, exist_ok=True) + directory = arguments.output + node = arguments.node + directory.mkdir(parents=True) + binding = import_module("pylibseekdb") + native_engine = Path(binding.__file__).with_name("seekdb") + with native_engine.open("rb") as engine_file: + native_digest = hashlib.file_digest(engine_file, "sha256").hexdigest() (directory / "environment.json").write_text( json.dumps( { "platform": platform.platform(), "python": sys.version, "pylibseekdb": version("pylibseekdb"), - "node": NODE, + "seekdb_engine": str(native_engine), + "seekdb_engine_sha256": native_digest, + "node": node, + "scenario": arguments.scenario, + "restart": arguments.restart, }, indent=2, + ensure_ascii=False, ) + "\n" ) for attempt in range(1, arguments.attempts + 1): attempt_directory = directory / f"attempt-{attempt:02d}" - if not run_attempt(attempt_directory, arguments.timeout): + if not run_attempt( + attempt_directory, + arguments.timeout, + node=node, + restart=arguments.restart, + fixture=arguments.fixture, + keep_failed_storage=arguments.keep_failed_storage, + ): return 1 # Keep a complete successful baseline and every failure. Other successful # repeats retain their Python trace, lifecycle and test result only. diff --git a/tests/builtin/persistence/test_mysql_migration_checks.py b/tests/builtin/persistence/test_mysql_migration_checks.py index c0e4a7d6ce..cb50afac8e 100644 --- a/tests/builtin/persistence/test_mysql_migration_checks.py +++ b/tests/builtin/persistence/test_mysql_migration_checks.py @@ -30,7 +30,6 @@ from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, _expression_sql, mysql_inventory from powercontext.builtin.persistence.seekdb import SeekDBConfig -from tests.builtin.persistence.test_mysql_migration_runner import _seekdb_scenario_lifetime from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase _FIXTURE = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json" @@ -179,9 +178,7 @@ def reflect(connection, _identity, guard): [("a % 2 = 0", "INTEGER", 2, 3), ("a = '100%s%%'", "VARCHAR(32)", "100%s%%", "different")], ids=["remainder", "percent_literal"], ) -def test_real_seekdb_percent_checks_validate_data_and_plan( - short_tmp_path, monkeypatch, expression, column_type, valid, invalid -): +def test_real_seekdb_percent_checks_validate_data_and_plan(short_tmp_path, expression, column_type, valid, invalid): pytest.importorskip("pylibseekdb") # Extend an isolated bundle, preserving the shipped layouts and their eligibility. directory = short_tmp_path / "bundle" @@ -222,23 +219,22 @@ def invalidate(connection, _identity, guard): with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): schema.validate_data(connection, "p0001") - with _seekdb_scenario_lifetime(monkeypatch): - fresh = MySQLMigrationRunner( - MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "fresh_seekdb")), - bundle, - evidence_directory=short_tmp_path / "fresh_evidence", - ) - fresh_plan = fresh.plan(backup_policy="skip") - fresh.apply( - plan_id=fresh_plan.plan_id, - accepted=True, - maintenance_confirmed=True, - backup_policy="skip", - accept_no_backup=True, - ) - assert fresh.verify().state == "ready" - connections.run(prepare, writable=True) - assert runner.plan().state == "migration_required" - connections.run(invalidate, writable=True) - with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): - runner.plan() + fresh = MySQLMigrationRunner( + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "fresh_seekdb")), + bundle, + evidence_directory=short_tmp_path / "fresh_evidence", + ) + fresh_plan = fresh.plan(backup_policy="skip") + fresh.apply( + plan_id=fresh_plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="skip", + accept_no_backup=True, + ) + assert fresh.verify().state == "ready" + connections.run(prepare, writable=True) + assert runner.plan().state == "migration_required" + connections.run(invalidate, writable=True) + with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): + runner.plan() diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 0357bbcf43..a81efdb2e8 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -27,6 +27,7 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime +from importlib import import_module from pathlib import Path from typing import Literal, cast from unittest.mock import Mock @@ -126,44 +127,19 @@ async def manage(*, create: bool) -> None: pytest.fail("OceanBase owned test database cleanup failed", pytrace=False) -@contextmanager -def _seekdb_scenario_lifetime(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: - binding = pytest.importorskip("pylibseekdb") - open_instance = binding.aopen - instances = {} - - async def open_with_scenario_lifetime(path: str): - # Keep a real native lease across this scenario's independent - # maintenance connections. Rapid last-client exits in seekdb 1.4 - # can strand a background DDL lock on restart. Engine restart and - # cross-process ownership have separate connection acceptance tests. - # Acquire lazily so inspecting a missing target still creates nothing. - if path not in instances: - instances[path] = await open_instance(path) - return await open_instance(path) - - monkeypatch.setattr(binding, "aopen", open_with_scenario_lifetime) - try: - yield - finally: - for instance in instances.values(): - instance.close() - - @pytest.fixture(params=("seekdb", "oceanbase"), ids=("seekdb", "oceanbase")) def backend( request: pytest.FixtureRequest, short_tmp_path: Path, caplog: pytest.LogCaptureFixture, - monkeypatch: pytest.MonkeyPatch, ) -> Iterator[_BackendTarget]: if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SQL_LOG") == "1": caplog.set_level(logging.INFO, logger="sqlalchemy.engine") if request.param == "seekdb": if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1": pytest.skip("real seekdb probe not enabled") - with _seekdb_scenario_lifetime(monkeypatch): - yield _BackendTarget(SeekDBConfig(path=short_tmp_path / "seekdb"), short_tmp_path / "evidence") + import_module("pylibseekdb") + yield _BackendTarget(SeekDBConfig(path=short_tmp_path / "seekdb"), short_tmp_path / "evidence") return value = os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL") if not value: @@ -1218,9 +1194,24 @@ def interrupt(journal, evidence) -> None: recovery = replacement.plan(backup_policy="manual") assert recovery.source_revision == "p0002" replacement = _acknowledge_known_completed_interruption(backend, recovery) - assert _apply(replacement, policy="manual").state == "ready" + upgrade = replacement._upgrade + recovered_p0003 = False + + def observe_recovery(connection, config, source, target, guard) -> None: + nonlocal recovered_p0003 + upgrade(connection, config, source, target, guard) + if target == "p0003": + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == "p0003" + recovered_p0003 = True + + with monkeypatch.context() as patch: + patch.setattr(replacement, "_upgrade", observe_recovery) + assert _apply(replacement, policy="manual").state == "ready" + assert recovered_p0003, "Recovery did not reach p0003" assert replacement.verify().revision == "p0004" - assert _read_state(backend)["content"] == [(b'{"content":"preserved"}',)] + state = _read_state(backend) + assert state["content"] == [(b'{"content":"preserved"}',)] + assert state["tags"] == [("Keep",)] def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledgement(backend: _BackendTarget) -> None: diff --git a/tests/builtin/persistence/test_mysql_vector_admission.py b/tests/builtin/persistence/test_mysql_vector_admission.py index 14a92a7721..223cf41164 100644 --- a/tests/builtin/persistence/test_mysql_vector_admission.py +++ b/tests/builtin/persistence/test_mysql_vector_admission.py @@ -69,7 +69,8 @@ def runner(selected=bundle, configured=configuration): monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) - def run(operation, *, writable=False): + def run(operation, *, writable=False, allow_missing_database=False): + del allow_missing_database # This fixture always opens its existing database. assert not writable, "Admission must fail before requesting a write connection" engine = create_engine(f"sqlite:///{database}") try: diff --git a/tests/builtin/test_code_publication.py b/tests/builtin/test_code_publication.py index 078203a5c7..f97d158449 100644 --- a/tests/builtin/test_code_publication.py +++ b/tests/builtin/test_code_publication.py @@ -308,7 +308,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") store.remove_generation("a" * 32, "binding", time.monotonic() + 30) @@ -490,7 +490,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") with pytest.raises(CodeError): store.remove_generation("not-hex", "binding", time.monotonic() + 30) @@ -538,7 +538,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") store.remove_generation("b" * 32, "binding", time.monotonic() + 30) diff --git a/tests/e2e/test_seekdb_restart_recovery.py b/tests/e2e/test_seekdb_restart_recovery.py new file mode 100644 index 0000000000..7c5b72671c --- /dev/null +++ b/tests/e2e/test_seekdb_restart_recovery.py @@ -0,0 +1,193 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native restart/recovery release gates; no additional instance lease is held.""" + +from __future__ import annotations + +import asyncio +import ctypes +import json +import os +import select +import shutil +import tempfile +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from importlib import import_module +from pathlib import Path + +import pytest +from sqlalchemy import Connection, text + +from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections +from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.seekdb import SeekDBConfig, profile + +pytestmark = pytest.mark.skipif( + os.environ.get("POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY") != "1", + reason="native seekdb restart/recovery release gate not enabled", +) + + +@pytest.fixture +def recovery_path() -> Iterator[Path]: + path = Path(tempfile.mkdtemp(prefix="pc-")) + try: + yield path + finally: + # The probe owns the parent directory and removes successful attempts. + # Keep failures even when the client timeout lets pytest exit normally. + if os.environ.get("POWERCONTEXT_SEEKDB_KEEP_FAILED_STORAGE") != "1": + shutil.rmtree(path) + + +def test_metadata_inspection_survives_last_client_restarts(recovery_path: Path) -> None: + import_module("pylibseekdb") + connections = MaintenanceConnections(SeekDBConfig(path=recovery_path / "seekdb")) + bundle = production_bundle() + schema = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text()) + + def seed(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + for statement in schema["mysql"]: + connection.exec_driver_sql(statement) + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('s','memory','m',1,:content)" + ), + {"content": b'{"content":"preserved"}'}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','memory','m',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_tags " + "(scope_id,family,artifact_id,target_type,target_id,tag_key_hash,tag_key,tag,assigned_at) " + "VALUES ('s','memory','m','artifact','m',UNHEX(REPEAT('00',32)),'keep','Keep','2026-09-28 12:00:00')" + ) + connection.commit() + + def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SHOW TRIGGERS").all() == [] + connection.commit() + + def adopt(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + inspect(connection, _identity, _guard) + connection.exec_driver_sql("START TRANSACTION") + connection.exec_driver_sql( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ) + connection.exec_driver_sql("INSERT INTO pc_schema_revision VALUES ('p0001')") + connection.commit() + + def verify(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").all() == [("p0001",)] + assert connection.exec_driver_sql("SELECT content FROM pc_artifacts").all() == [(b'{"content":"preserved"}',)] + assert connection.exec_driver_sql("SELECT tag FROM pc_artifact_tags").all() == [("Keep",)] + + connections.run(seed, writable=True) + connections.run(inspect) + connections.run(inspect) + connections.run(adopt, writable=True) + connections.run(verify) + + +@contextmanager +def _native_generation(binding, path: Path): + """Wait for Linux process exit, including when Python lacks os.pidfd_open.""" + instance = asyncio.run(profile._open_instance(binding, path)) + try: + pid = int((path / "run/seekdb.pid").read_text().strip()) + pidfd_open = ctypes.CDLL(None, use_errno=True).pidfd_open + pidfd_open.argtypes = (ctypes.c_int, ctypes.c_uint) + pidfd_open.restype = ctypes.c_int + descriptor = pidfd_open(pid, 0) + if descriptor < 0: + raise OSError(ctypes.get_errno(), "could not open native pidfd") # noqa: TRY301 -- close on setup failure + except BaseException: + instance.close() + raise + try: + yield instance + finally: + try: + instance.close() + assert select.select([descriptor], [], [], 10)[0], "native process did not exit after last-client close" + finally: + os.close(descriptor) + + +def test_saved_fault_recovers_and_remains_usable_after_reopen(recovery_path: Path) -> None: + binding = import_module("pylibseekdb") + pymysql = import_module("pymysql") + configured = os.environ.get("POWERCONTEXT_SEEKDB_RECOVERY_FIXTURE") + if not configured: + pytest.fail("enabled recovery acceptance requires an immutable native fault fixture directory", pytrace=False) + source = Path(configured).resolve() + assert (source / "etc/seekdb.data_version.bin").is_file(), "unrecognized native fault fixture" + assert (source / "store/sstable/meta.db").is_file(), "unrecognized native fault fixture" + path = recovery_path / "seekdb" + # Open only the disposable clone, including for startup/recovery writes. + shutil.copytree( + source, + path, + ignore=lambda directory, _names: ["run"] if Path(directory) == source else [], + ) + + def verify(connection) -> None: + with connection.cursor() as cursor: + cursor.execute("SELECT version_num FROM pc_schema_revision") + assert cursor.fetchall() == (("p0001",),) + cursor.execute("SELECT content FROM pc_artifacts") + assert cursor.fetchall() == ((b'{"content":"preserved"}',),) + cursor.execute("SELECT tag FROM pc_artifact_tags") + assert cursor.fetchall() == (("Keep",),) + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + with connection.cursor() as cursor: + cursor.execute( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ) + cursor.execute("INSERT INTO pc_schema_revision VALUES ('p0001')") + connection.commit() + verify(connection) + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + verify(connection) + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE pc_recovery_write_check (id INT PRIMARY KEY)") + cursor.execute("INSERT INTO pc_recovery_write_check VALUES (1)") + connection.commit() + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + verify(connection) + with connection.cursor() as cursor: + cursor.execute("SELECT id FROM pc_recovery_write_check") + assert cursor.fetchall() == ((1,),) diff --git a/tests/e2e/test_topic_memory_recovery_migration.py b/tests/e2e/test_topic_memory_recovery_migration.py index c0d84cc18b..d463043029 100644 --- a/tests/e2e/test_topic_memory_recovery_migration.py +++ b/tests/e2e/test_topic_memory_recovery_migration.py @@ -22,7 +22,7 @@ import os import shutil import sqlite3 -from contextlib import closing, nullcontext +from contextlib import closing import pytest from sqlalchemy import Engine, event @@ -41,7 +41,6 @@ from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService from powercontext.cli.app import create_cli from powercontext.server.cli import app -from tests.builtin.persistence.test_mysql_migration_runner import _seekdb_scenario_lifetime from tests.builtin.runtime.test_topic_memory_processing import _repositories from tests.builtin.runtime.test_topic_memory_scope import BINDING, _capture from tests.builtin.runtime.test_topic_memory_terminal_frontier import _budget, _seed @@ -587,7 +586,7 @@ async def reopen(): @pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) @pytest.mark.parametrize("interrupted", [False, True]) def test_real_mysql_recovery_upgrade_preserves_receipt_and_resumes_registered_ddl_prefix( - backend, interrupted, short_tmp_path, tmp_path, monkeypatch + backend, interrupted, short_tmp_path, tmp_path ): async def scenario(): async with _isolated_database(backend, short_tmp_path) as (config, open_profile): @@ -661,8 +660,4 @@ def stop_during_ddl(_connection, _cursor, statement, _parameters, _context, _exe ) ).one() == (3, 9, 1000) - # Exercise committed DDL recovery across independent connections without - # triggering seekdb 1.4's rapid last-client-close runtime lock stall. - # Native restart/exclusion acceptance remains in the connection tests. - with _seekdb_scenario_lifetime(monkeypatch) if backend == "seekdb" else nullcontext(): - asyncio.run(scenario()) + asyncio.run(scenario()) diff --git a/tests/test_ops.py b/tests/test_ops.py index d5c6666d2e..60dcbe19cb 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -507,6 +507,7 @@ def emit(message): # owns the worker whose exit releases both locks. bootstrap = """import runpy, site, sys packages, repository, script, *arguments = sys.argv[1:] +sys.path.insert(0, packages) site.addsitedir(packages) sys.path.insert(0, repository) sys.argv = [script, *arguments] diff --git a/tests/test_seekdb_migration_probe.py b/tests/test_seekdb_migration_probe.py new file mode 100644 index 0000000000..4190609c06 --- /dev/null +++ b/tests/test_seekdb_migration_probe.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A skipped test or retained engine must not certify native recovery.""" + +import json +from pathlib import Path + +import pytest + +from scripts.probe_seekdb_migration_lock import restart_evidence, successful_report + + +@pytest.mark.parametrize("outcome", ["", "", "", ""]) +def test_native_acceptance_requires_one_passing_case(tmp_path: Path, outcome: str) -> None: + report = tmp_path / "junit.xml" + report.write_text(f"{outcome}") + assert successful_report(report, 0) is (outcome == "") + assert not successful_report(report, 1) + report.write_text("") + assert not successful_report(report, 0) + + +@pytest.mark.parametrize("pids, accepted", [([], False), ([101], False), ([101, 101], False), ([101, 102], True)]) +def test_restart_acceptance_rejects_an_engine_kept_alive(tmp_path: Path, pids: list[int], accepted: bool) -> None: + events = [{"event": "opened", "path": "/synthetic/seekdb", "native_pid": pid} for pid in pids] + (tmp_path / "lifecycle.jsonl").write_text("\n".join(json.dumps(event) for event in events)) + assert restart_evidence(tmp_path)["passed"] is accepted diff --git a/tox.ini b/tox.ini index 3cab314540..bccaa0da49 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,10 @@ python = [testenv] passenv = PYTHON_VERSION allowlist_externals = uv +setenv = + UV_PROJECT_ENVIRONMENT={envdir} + COVERAGE_FILE={envtmpdir}/.coverage commands = - uv sync --python {envpython} - uv run python -m pytest --doctest-modules tests --cov --cov-config=pyproject.toml --cov-report=xml - ty check + uv sync --locked --python {envpython} + uv run --no-sync python -m pytest --doctest-modules tests --cov=powercontext --cov-config=pyproject.toml --cov-report=xml:{envtmpdir}/coverage.xml {posargs} + uv run --no-sync ty check From 7fb5e0f3e16c85933f783a38eee28095d874ca72 Mon Sep 17 00:00:00 2001 From: lkx Date: Sat, 31 Oct 2026 18:49:34 +0800 Subject: [PATCH 2/2] fix(seekdb): confirm process exit in restart probes --- .github/ci-paths.toml | 1 + .github/workflows/seekdb-lock-observation.yml | 2 +- .github/workflows/seekdb-lock-probe.yml | 9 ++- docs/en/docs/operate/database-migrations.md | 20 +++--- docs/zh/docs/operate/database-migrations.md | 16 +++-- scripts/probe_seekdb_migration_lock.py | 35 ++++++++-- tests/e2e/test_seekdb_restart_recovery.py | 37 +++++------ tests/seekdb_support.py | 60 +++++++++++++++++ tests/test_seekdb_migration_probe.py | 65 ++++++++++++++++++- 9 files changed, 202 insertions(+), 43 deletions(-) create mode 100644 tests/seekdb_support.py diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index f3bdb066be..9be4937872 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -28,6 +28,7 @@ tests = [ ] evaluation = ["evaluation/**"] database = [ + "tests/seekdb_support.py", "tests/builtin/persistence/**", "tests/builtin/*native_code*", "tests/builtin/runtime/*recall*", "tests/builtin/runtime/test_statistics.py", "tests/builtin/runtime/test_model_usage_recorder.py", "tests/builtin/runtime/test_prepared_context.py", "tests/e2e/*atomic*", "tests/e2e/*mysql*", diff --git a/.github/workflows/seekdb-lock-observation.yml b/.github/workflows/seekdb-lock-observation.yml index 43a8f889ec..fc281623cf 100644 --- a/.github/workflows/seekdb-lock-observation.yml +++ b/.github/workflows/seekdb-lock-observation.yml @@ -36,7 +36,7 @@ jobs: # experiment records snapshots every 45 seconds for up to 20 minutes # per attempt, including beyond the observed 1000-second SQL timeout. uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ - --restart --attempts 12 --timeout 1200 --output .powercontext/seekdb-lock-observation + --attempts 12 --timeout 1200 --output .powercontext/seekdb-lock-observation - name: Upload observation evidence if: always() diff --git a/.github/workflows/seekdb-lock-probe.yml b/.github/workflows/seekdb-lock-probe.yml index ad984f873e..08ded81728 100644 --- a/.github/workflows/seekdb-lock-probe.yml +++ b/.github/workflows/seekdb-lock-probe.yml @@ -8,6 +8,7 @@ on: - tests/builtin/persistence/test_mysql_migration_runner.py - tests/e2e/test_seekdb_restart_recovery.py - tests/test_seekdb_migration_probe.py + - tests/seekdb_support.py - src/powercontext/builtin/persistence/** - pyproject.toml - uv.lock @@ -37,9 +38,15 @@ jobs: with: python-version: ${{ matrix.python }} - name: Reproduce migration lifecycle with complete native evidence + env: + SEEKDB_PROBE_SCENARIO: ${{ matrix.scenario }} run: | + restart_args=() + if [ "$SEEKDB_PROBE_SCENARIO" = restart-recovery ]; then + restart_args+=(--restart) + fi uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ - --scenario '${{ matrix.scenario }}' --restart --attempts 12 --timeout 180 + --scenario "$SEEKDB_PROBE_SCENARIO" "${restart_args[@]}" --attempts 12 --timeout 180 - name: Upload native logs, lifecycle and live transaction snapshots if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index f9de1fbda8..95ccc7edf5 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -444,22 +444,26 @@ Run the following from the repository root, separately under Python 3.11 and Pyt ```bash uv sync --locked --extra seekdb uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ - --scenario migration --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-migration + --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart ``` Use a new output directory for each run. The migration scenario selects -`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]`. The restart scenario checks metadata -inspection, commit, last-client shutdown, baseline DDL and preserved content and tags after reopening. Each attempt -must report exactly one passing test, with no skip, error or timeout. `--restart` also requires multiple distinct native -process IDs for the same directory. Evidence includes the Python and binding versions, native binary SHA-256, instance +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` with its natural client lifecycle. Releasing a +client does not imply that the native process has already exited; a rapid reopen may reuse it. The restart scenario +checks metadata inspection, commit, confirmed last-client shutdown, baseline DDL and preserved content and tags after +reopening. Its test waits for kernel process-exit notification even when invoked without the probe. Each attempt must +report exactly one passing test, with no skip, error or timeout. `--restart` explicitly waits up to 10 seconds for native +exit on each close, then requires distinct process IDs and an exit record before each subsequent open. A timeout fails +the attempt; it does not retry. Evidence includes the Python and binding versions, native binary SHA-256, instance lifetimes, SQL trace, compressed native logs and live transaction snapshots for slow attempts. The separate extended observation workflow is diagnostic; its longer deadline does not replace the ordinary acceptance deadline. -The frozen-fixture gate runs on Linux and observes kernel process-exit notifications before each reopen; -it does not use a fixed startup delay. For recovery acceptance, first reproduce the stall with an isolated `1.4.0.post1` environment and -`--scenario restart-recovery --keep-failed-storage`. Stop its engine before freezing the failed `tmp/pc-*/seekdb` +Both explicit restart gates run on Linux and observe kernel process-exit notifications before each reopen; +neither uses a fixed startup delay. For recovery acceptance, first reproduce the stall with an isolated +`1.4.0.post1` environment and `--scenario restart-recovery --keep-failed-storage`. +Stop its engine before freezing the failed `tmp/pc-*/seekdb` directory. Keep the original immutable. This scenario creates synthetic data in the `powercontext` database, including the `preserved` content and `Keep` tag expected by the fixture test. A healthy database or a fixture with different schema or sentinels cannot establish recovery of this defect. Compare the old and fixed engines on separate copies of diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 34bbd466c4..c1cac66933 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -345,19 +345,23 @@ powercontext server db-migrate verify --env-file oceanbase.env \ ```bash uv sync --locked --extra seekdb uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ - --scenario migration --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-migration + --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart ``` 每次运行使用新的输出目录。migration 场景选择原 issue 的 -`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` 节点；restart-recovery 场景检查元数据查询、 -提交、最后一个客户端关闭、基线 DDL，以及重开后内容和标签的保留。每次必须恰好一个测试通过，跳过、报错和超时均不算通过。 -`--restart` 还要求同一目录经历多个不同的原生进程 PID。证据包括 Python 和绑定版本、原生二进制 SHA-256、实例生命周期、 +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` 节点，保留自然客户端生命周期。释放客户端不代表 +原生进程已经退出，紧接着打开可能复用该进程。restart-recovery 场景检查元数据查询、提交、确认最后一个客户端关闭后 +进程退出、基线 DDL，以及重开后内容和标签的保留；即使不通过探针调用，该测试也会等待内核进程退出通知。 +每次必须恰好一个测试通过，跳过、报错和超时均不算通过。`--restart` 在每次关闭时最多等待 10 秒确认原生进程退出， +再要求各代 PID 不同，且下一次打开之前存在退出记录。等待超时直接失败，不自动重试。 +证据包括 Python 和绑定版本、原生二进制 SHA-256、实例生命周期、 SQL 日志、压缩后的原生日志，以及慢执行时的事务快照。独立的延长观察工作流仅用于诊断，不能用它的较长时限替代正常验收时限。 -故障库门禁在 Linux 上执行，每次重开前通过内核退出事件确认上一代进程已结束，不使用固定启动等待。 -故障库验收应先在隔离的 `1.4.0.post1` 环境中，使用 `--scenario restart-recovery --keep-failed-storage` 复现阻塞。 +两个显式重开验收均在 Linux 上执行，每次重开前通过内核退出事件确认上一代进程已结束，不使用固定启动等待。 +故障库验收应先在隔离的 `1.4.0.post1` 环境中， +使用 `--scenario restart-recovery --keep-failed-storage` 复现阻塞。 停止引擎后，冻结失败输出中的 `tmp/pc-*/seekdb` 目录，并保持原件不变。此场景在 `powercontext` 数据库中创建合成数据， 包括故障库测试要求的 `preserved` 内容和 `Keep` 标签。健康库、结构或哨兵数据不同的库，不能用于证明该缺陷已恢复。 使用同一冻结故障库的独立副本，对比旧引擎与修复后的引擎： diff --git a/scripts/probe_seekdb_migration_lock.py b/scripts/probe_seekdb_migration_lock.py index cf403b3730..31057ac59e 100644 --- a/scripts/probe_seekdb_migration_lock.py +++ b/scripts/probe_seekdb_migration_lock.py @@ -64,8 +64,10 @@ def record(directory: Path, event: str, **values: object) -> None: def pytest_sessionstart(session: pytest.Session) -> None: """Optional pytest plugin: observe only this subprocess's native instance lifetime.""" from powercontext.builtin.persistence.seekdb import profile + from tests.seekdb_support import RestartingInstance directory = Path(os.environ["POWERCONTEXT_SEEKDB_PROBE"]) + restart = os.environ.get("POWERCONTEXT_SEEKDB_REQUIRE_RESTART") == "1" original = profile._open_instance class ObservedInstance: @@ -81,6 +83,8 @@ def close(self) -> None: record(directory, "closing", path=str(self.path), native_pid=self.native_pid) try: self.instance.close() + if restart: + record(directory, "native_exited", path=str(self.path), native_pid=self.native_pid) finally: record(directory, "close_returned", path=str(self.path), native_pid=self.native_pid) @@ -92,6 +96,8 @@ async def observed_open(module: ModuleType, path: Path) -> profile._SeekDBInstan except (OSError, ValueError): instance.close() raise + if restart: + instance = RestartingInstance(instance, native_pid) options = dict(instance.connection_options()) # Embedded scratch instance only. Never capture credentials or external connection settings. socket = options.get("unix_socket") @@ -207,16 +213,24 @@ def stop_process(process: subprocess.Popen[bytes]) -> None: def restart_evidence(directory: Path) -> dict[str, Any]: - """Require repeated native starts without reusing a process for the same target.""" + """Require confirmed exit before each subsequent native generation opens.""" lifecycle = directory / "lifecycle.jsonl" events = [json.loads(line) for line in lifecycle.read_text().splitlines()] if lifecycle.exists() else [] processes: dict[str, list[int]] = {} + running: dict[str, int] = {} + valid = True for event in events: if event["event"] == "opened": - processes.setdefault(event["path"], []).append(event["native_pid"]) - passed = any(len(pids) > 1 for pids in processes.values()) and all( - all(pid > 0 for pid in pids) and len(set(pids)) == len(pids) for pids in processes.values() - ) + path, pid = event["path"], event["native_pid"] + pids = processes.setdefault(path, []) + valid = valid and path not in running and pid > 0 and pid not in pids + pids.append(pid) + running[path] = pid + elif event["event"] == "native_exited": + path, pid = event["path"], event["native_pid"] + valid = valid and running.get(path) == pid + running.pop(path, None) + passed = valid and not running and any(len(pids) > 1 for pids in processes.values()) return {"passed": passed, "processes": processes} @@ -236,6 +250,7 @@ def run_attempt( os.environ, TMPDIR=str(temporary), POWERCONTEXT_SEEKDB_PROBE=str(directory), + POWERCONTEXT_SEEKDB_REQUIRE_RESTART=str(int(restart)), POWERCONTEXT_TEST_MIGRATION_SEEKDB="1", POWERCONTEXT_TEST_MIGRATION_SQL_LOG="1", POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY="1", @@ -343,7 +358,9 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument( "--keep-failed-storage", action="store_true", help="Retain failed scratch storage for recovery acceptance." ) - parser.add_argument("--restart", action="store_true", help="Require distinct native process IDs across reopenings.") + parser.add_argument( + "--restart", action="store_true", help="Wait for Linux native process exit before each reopen and verify it." + ) parser.add_argument("--timeout", type=positive_integer, default=180) arguments = parser.parse_args() if arguments.snapshot: @@ -374,7 +391,11 @@ def main() -> int: node = arguments.node directory.mkdir(parents=True) binding = import_module("pylibseekdb") - native_engine = Path(binding.__file__).with_name("seekdb") + binding_file = binding.__file__ + if binding_file is None: + message = "The SeekDB binding has no module file; cannot locate its native engine" + raise RuntimeError(message) + native_engine = Path(binding_file).with_name("seekdb") with native_engine.open("rb") as engine_file: native_digest = hashlib.file_digest(engine_file, "sha256").hexdigest() (directory / "environment.json").write_text( diff --git a/tests/e2e/test_seekdb_restart_recovery.py b/tests/e2e/test_seekdb_restart_recovery.py index 7c5b72671c..7a3baa3038 100644 --- a/tests/e2e/test_seekdb_restart_recovery.py +++ b/tests/e2e/test_seekdb_restart_recovery.py @@ -17,10 +17,8 @@ from __future__ import annotations import asyncio -import ctypes import json import os -import select import shutil import tempfile from collections.abc import Callable, Iterator @@ -34,6 +32,7 @@ from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections from powercontext.builtin.persistence.migrations.deployment import production_bundle from powercontext.builtin.persistence.seekdb import SeekDBConfig, profile +from tests.seekdb_support import RestartingInstance pytestmark = pytest.mark.skipif( os.environ.get("POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY") != "1", @@ -41,6 +40,22 @@ ) +@pytest.fixture(autouse=True) +def native_restarts(monkeypatch: pytest.MonkeyPatch) -> None: + original = profile._open_instance + + async def open_generation(binding, path: Path): + instance = await original(binding, path) + try: + pid = int((path / "run/seekdb.pid").read_text().strip()) + except (OSError, ValueError): + instance.close() + raise + return RestartingInstance(instance, pid) + + monkeypatch.setattr(profile, "_open_instance", open_generation) + + @pytest.fixture def recovery_path() -> Iterator[Path]: path = Path(tempfile.mkdtemp(prefix="pc-")) @@ -111,27 +126,11 @@ def verify(connection: Connection | None, _identity: BackendIdentity, _guard: Ca @contextmanager def _native_generation(binding, path: Path): - """Wait for Linux process exit, including when Python lacks os.pidfd_open.""" instance = asyncio.run(profile._open_instance(binding, path)) - try: - pid = int((path / "run/seekdb.pid").read_text().strip()) - pidfd_open = ctypes.CDLL(None, use_errno=True).pidfd_open - pidfd_open.argtypes = (ctypes.c_int, ctypes.c_uint) - pidfd_open.restype = ctypes.c_int - descriptor = pidfd_open(pid, 0) - if descriptor < 0: - raise OSError(ctypes.get_errno(), "could not open native pidfd") # noqa: TRY301 -- close on setup failure - except BaseException: - instance.close() - raise try: yield instance finally: - try: - instance.close() - assert select.select([descriptor], [], [], 10)[0], "native process did not exit after last-client close" - finally: - os.close(descriptor) + instance.close() def test_saved_fault_recovers_and_remains_usable_after_reopen(recovery_path: Path) -> None: diff --git a/tests/seekdb_support.py b/tests/seekdb_support.py new file mode 100644 index 0000000000..0b8b75de34 --- /dev/null +++ b/tests/seekdb_support.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Linux process-exit ownership for native SeekDB acceptance tests.""" + +from __future__ import annotations + +import ctypes +import os +import select +from typing import Any + + +class RestartingInstance: + """Release a client and require its native process to exit before returning.""" + + def __init__(self, instance: Any, native_pid: int, *, timeout: float = 10) -> None: + self.instance = instance + self.timeout = timeout + self.descriptor: int | None = None + try: + if native_pid <= 0: + raise ValueError("native process ID must be positive") # noqa: TRY003, TRY301 + pidfd_open = ctypes.CDLL(None, use_errno=True).pidfd_open + pidfd_open.argtypes = (ctypes.c_int, ctypes.c_uint) + pidfd_open.restype = ctypes.c_int + descriptor = pidfd_open(native_pid, 0) + if descriptor < 0: + raise OSError(ctypes.get_errno(), "could not open native pidfd") # noqa: TRY301 + self.descriptor = descriptor + except BaseException: + instance.close() + raise + + def connection_options(self) -> Any: + return self.instance.connection_options() + + def close(self) -> None: + descriptor = self.descriptor + if descriptor is None: + return + self.descriptor = None + try: + self.instance.close() + if not select.select([descriptor], [], [], self.timeout)[0]: + message = "native process did not exit after last-client close" + raise TimeoutError(message) + finally: + os.close(descriptor) diff --git a/tests/test_seekdb_migration_probe.py b/tests/test_seekdb_migration_probe.py index 4190609c06..85296fabfd 100644 --- a/tests/test_seekdb_migration_probe.py +++ b/tests/test_seekdb_migration_probe.py @@ -15,11 +15,17 @@ """A skipped test or retained engine must not certify native recovery.""" import json +import subprocess +import sys +from collections.abc import Iterator from pathlib import Path +from types import ModuleType, SimpleNamespace import pytest +from scripts import probe_seekdb_migration_lock as probe from scripts.probe_seekdb_migration_lock import restart_evidence, successful_report +from tests.seekdb_support import RestartingInstance @pytest.mark.parametrize("outcome", ["", "", "", ""]) @@ -34,6 +40,63 @@ def test_native_acceptance_requires_one_passing_case(tmp_path: Path, outcome: st @pytest.mark.parametrize("pids, accepted", [([], False), ([101], False), ([101, 101], False), ([101, 102], True)]) def test_restart_acceptance_rejects_an_engine_kept_alive(tmp_path: Path, pids: list[int], accepted: bool) -> None: - events = [{"event": "opened", "path": "/synthetic/seekdb", "native_pid": pid} for pid in pids] + events = [ + {"event": event, "path": "/synthetic/seekdb", "native_pid": pid} + for pid in pids + for event in ("opened", "native_exited") + ] (tmp_path / "lifecycle.jsonl").write_text("\n".join(json.dumps(event) for event in events)) assert restart_evidence(tmp_path)["passed"] is accepted + + +@pytest.mark.parametrize("exit_before_reopen", [False, True]) +def test_different_pids_cannot_certify_restart_without_confirmed_exit(tmp_path: Path, exit_before_reopen: bool) -> None: + events = [{"event": "opened", "path": "/synthetic/seekdb", "native_pid": 101}] + exited = {"event": "native_exited", "path": "/synthetic/seekdb", "native_pid": 101} + if exit_before_reopen: + events.append(exited) + events.append({"event": "opened", "path": "/synthetic/seekdb", "native_pid": 102}) + if not exit_before_reopen: + events.append(exited) + events.append({"event": "native_exited", "path": "/synthetic/seekdb", "native_pid": 102}) + (tmp_path / "lifecycle.jsonl").write_text("\n".join(json.dumps(event) for event in events)) + assert not restart_evidence(tmp_path)["passed"] + + +def test_probe_rejects_a_binding_without_a_module_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + binding = ModuleType("pylibseekdb") + binding.__file__ = None + monkeypatch.setattr(probe, "import_module", lambda _name: binding) + monkeypatch.setattr(sys, "argv", ["probe", "--output", str(tmp_path / "probe")]) + with pytest.raises(RuntimeError, match="no module file"): + probe.main() + + +@pytest.fixture +def owned_process() -> Iterator[subprocess.Popen[bytes]]: + with subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdin.buffer.read()"], stdin=subprocess.PIPE + ) as process: + try: + yield process + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_last_client_close_waits_for_process_exit(owned_process: subprocess.Popen[bytes]) -> None: + assert owned_process.stdin is not None + instance = RestartingInstance(SimpleNamespace(close=owned_process.stdin.close), owned_process.pid) + instance.close() + assert owned_process.poll() == 0 + instance.close() + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_last_client_close_rejects_a_process_that_stays_alive(owned_process: subprocess.Popen[bytes]) -> None: + instance = RestartingInstance(SimpleNamespace(close=lambda: None), owned_process.pid, timeout=0) + with pytest.raises(TimeoutError, match="did not exit"): + instance.close() + assert owned_process.poll() is None