test(seekdb): enforce native restart and recovery acceptance

.github/ci-paths.toml

@@ -31,10 +31,11 @@ tests = [
 ]
 evaluation = ["evaluation/**"]
 database = [
+  "tests/seekdb_support.py",
   "tests/builtin/persistence/**", "tests/builtin/*native_code*", "tests/builtin/runtime/*recall*",
   "tests/builtin/runtime/test_statistics.py", "tests/builtin/runtime/test_model_usage_recorder.py",
   "tests/builtin/runtime/test_prepared_context.py", "tests/e2e/*atomic*", "tests/e2e/*mysql*",
-  "tests/e2e/*native_code*", "tests/e2e/test_recall_sufficiency_gate.py",
+  "tests/e2e/*native_code*", "tests/e2e/test_seekdb_restart_recovery.py", "tests/e2e/test_recall_sufficiency_gate.py",
   "tests/e2e/test_context_text_assembly.py", "tests/e2e/test_scope_search_migration.py",
   "scripts/run_atomic_memory_clean_migration.py",
   ".github/workflows/e2e-harness.yml", ".github/actions/setup-upgrade-oceanbase/**",
@@ -114,7 +115,13 @@ release_schema = [
   "tests/release_upgrades/test_full_server.py", ".github/workflows/capture-release-schema.yml",
   ".github/actions/setup-upgrade-oceanbase/**",
 ]
-seekdb_lock = ["scripts/probe_seekdb_migration_lock.py", ".github/workflows/seekdb-lock-probe.yml", ".github/workflows/seekdb-lock-observation.yml"]
+seekdb_lock = [
+  "scripts/probe_seekdb_migration_lock.py", ".github/workflows/seekdb-lock-probe.yml",
+  ".github/workflows/seekdb-lock-observation.yml", "tests/seekdb_support.py",
+  "tests/test_seekdb_migration_probe.py", "tests/e2e/test_seekdb_restart_recovery.py",
+  "tests/builtin/persistence/test_mysql_migration_runner.py",
+  "src/powercontext/builtin/persistence/**", "pyproject.toml", "uv.lock",
+]
 skill_guidance = [
   "evaluation/skills/skill-up/**",
   "integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/**",

.github/workflows/master.yml

@@ -477,6 +477,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.14"]
+        migration-shard: [0, 1]
     steps:
       - name: Check out
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -490,9 +491,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Test code indexing with embedded seekdb
+        if: matrix.migration-shard == 0
         run: make code-seekdb-test
 
       - name: Test database bootstrap with embedded seekdb without skips
+        if: matrix.migration-shard == 0
         # Two 360s native-process deadlines still leave room for the remaining
         # bootstrap cases and diagnostic upload before the step deadline.
         timeout-minutes: 20
@@ -518,6 +521,7 @@ jobs:
           PYCODE
 
       - name: Test batched archive validation with embedded seekdb
+        if: matrix.migration-shard == 0
         timeout-minutes: 5
         run: |
           uv run --locked --extra seekdb python -c "import pylibseekdb"
@@ -526,10 +530,11 @@ jobs:
 
       - name: Test unified migrations with embedded seekdb
         if: ${{ !cancelled() && steps.seekdb_setup.outcome == 'success' }}
-        # Full-Server cases run in separate processes and exceed fifteen minutes
-        # in aggregate; each runner keeps its 180-second deadline below.
+        # Last-client shutdown adds native startup work. Partition runner nodes;
+        # every node keeps its 180-second deadline.
         timeout-minutes: 25
         env:
+          SEEKDB_MIGRATION_SHARD: ${{ matrix.migration-shard }}
           POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1"
           POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1"
           PYTHONUNBUFFERED: "1"
@@ -549,16 +554,18 @@ jobs:
           }
           trap collect_diagnostics EXIT
           df -h . /tmp
-          uv run --locked --extra seekdb pytest -vv \
-            -o faulthandler_timeout=120 --durations=20 \
-            tests/builtin/persistence/test_database_migration_backends.py \
-            tests/builtin/persistence/test_database_backup_providers.py \
-            tests/builtin/persistence/test_database_migration_connections.py \
-            tests/e2e/test_topic_memory_recovery_migration.py \
-            tests/builtin/persistence/test_mysql_migration_checks.py \
-            tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \
-            2>&1 | tee seekdb-migrations.log
-          uv run --locked --extra seekdb pytest --collect-only -q --color=no \
+          if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then
+            uv run --locked --extra seekdb --extra code pytest -vv \
+              -o faulthandler_timeout=120 --durations=20 \
+              tests/builtin/persistence/test_database_migration_backends.py \
+              tests/builtin/persistence/test_database_backup_providers.py \
+              tests/builtin/persistence/test_database_migration_connections.py \
+              tests/e2e/test_topic_memory_recovery_migration.py \
+              tests/builtin/persistence/test_mysql_migration_checks.py \
+              tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \
+              2>&1 | tee seekdb-migrations.log
+          fi
+          uv run --locked --extra seekdb --extra code pytest --collect-only -q --color=no \
             tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \
             2>&1 | tee seekdb-runner-collection.log | tee -a seekdb-migrations.log
           mapfile -t seekdb_runner_nodes < <(
@@ -569,33 +576,39 @@ jobs:
             echo "No seekdb migration runner tests were collected" >&2
             exit 1
           fi
-          for seekdb_runner_node in "${seekdb_runner_nodes[@]}"; do
-            uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \
+          for index in "${!seekdb_runner_nodes[@]}"; do
+            if [ "$((index % 2))" -ne "$SEEKDB_MIGRATION_SHARD" ]; then
+              continue
+            fi
+            seekdb_runner_node="${seekdb_runner_nodes[$index]}"
+            uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \
               "$seekdb_runner_node" \
               2>&1 | tee -a seekdb-migrations.log
           done
           # These are required independent runs, not retries: any failure stops
           # the job. Exercise the lifecycle race on both supported matrix ends.
-          for repetition in 2 3; do
-            echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log
-            uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \
-              'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \
-              2>&1 | tee -a seekdb-migrations.log
-          done
+          if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then
+            for repetition in 2 3; do
+              echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log
+              uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \
+                'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \
+                2>&1 | tee -a seekdb-migrations.log
+            done
+          fi
 
       - name: Upload seekdb bootstrap diagnostics
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
         with:
-          name: seekdb-bootstrap-${{ matrix.python-version }}
+          name: seekdb-bootstrap-${{ matrix.python-version }}-${{ matrix.migration-shard }}
           path: .powercontext/database-bootstrap-ci/
           if-no-files-found: ignore
 
       - name: Upload seekdb migration diagnostics
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
         with:
-          name: seekdb-migrations-${{ matrix.python-version }}
+          name: seekdb-migrations-${{ matrix.python-version }}-${{ matrix.migration-shard }}
           path: |
             seekdb-migrations.log
             seekdb-native.log

.github/workflows/seekdb-lock-probe.yml

@@ -28,22 +28,41 @@ jobs:
     needs: changes
     if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').seekdb_lock) }}
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11', '3.14']
+        scenario: [migration, restart-recovery]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
           persist-credentials: false
       - uses: ./.github/actions/setup-python-env
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python }}
       - name: Reproduce migration lifecycle with complete native evidence
-        run: uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py --attempts 12
+        env:
+          SEEKDB_PROBE_SCENARIO: ${{ matrix.scenario }}
+        run: |
+          restart_args=()
+          if [ "$SEEKDB_PROBE_SCENARIO" = restart-recovery ]; then
+            restart_args+=(--restart)
+          fi
+          uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+            --scenario "$SEEKDB_PROBE_SCENARIO" "${restart_args[@]}" --attempts 12 --timeout 180
       - name: Upload native logs, lifecycle and live transaction snapshots
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
         with:
-          name: seekdb-lock-probe-${{ github.sha }}
-          path: .powercontext/seekdb-lock-probe
+          name: seekdb-lock-probe-${{ matrix.python }}-${{ matrix.scenario }}-${{ github.sha }}
+          path: |
+            .powercontext/seekdb-lock-probe/environment.json
+            .powercontext/seekdb-lock-probe/attempt-*/*.json
+            .powercontext/seekdb-lock-probe/attempt-*/*.jsonl
+            .powercontext/seekdb-lock-probe/attempt-*/*.xml
+            .powercontext/seekdb-lock-probe/attempt-*/*.log
+            .powercontext/seekdb-lock-probe/attempt-*/*.gz
           include-hidden-files: true
           if-no-files-found: error
           retention-days: 7

docs/en/docs/operate/database-migrations.md

@@ -435,6 +435,53 @@ Use the latest plan's `plan_id` and `coordination.pending_run_id`. Reconfirm the
 the original recovery point with a backup of partial state. Normal upgrades still use one confirmation. Recovery requires
 the additional explicit DBA declaration; PC never invents it when showing the plan.
 
+## Native SeekDB restart and recovery acceptance
+
+The embedded engine must use `pylibseekdb>=1.4.0.post2,<2`. The native transaction recovery fix is supplied by
+that dependency. The migration checks retain `SHOW TRIGGERS` and release the last client between operations.
+Run the following from the repository root, separately under Python 3.11 and Python 3.14:
+
+```bash
+uv sync --locked --extra seekdb
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart
+```
+
+Use a new output directory for each run. The migration scenario selects
+`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` with its natural client lifecycle. Releasing a
+client does not imply that the native process has already exited; a rapid reopen may reuse it. The restart scenario
+checks metadata inspection, commit, confirmed last-client shutdown, baseline DDL and preserved content and tags after
+reopening. Its test waits for kernel process-exit notification even when invoked without the probe. Each attempt must
+report exactly one passing test, with no skip, error or timeout. `--restart` explicitly waits up to 10 seconds for native
+exit on each close, then requires distinct process IDs and an exit record before each subsequent open. A timeout fails
+the attempt; it does not retry. Evidence includes the Python and binding versions, native binary SHA-256, instance
+lifetimes, SQL trace, compressed native logs and live transaction snapshots for slow attempts. The separate extended
+observation workflow is diagnostic; its longer deadline does not replace the ordinary acceptance deadline.
+
+Both explicit restart gates run on Linux and observe kernel process-exit notifications before each reopen;
+neither uses a fixed startup delay. The tests prefer Python's `os.pidfd_open`, then the optional libc wrapper.
+When both are absent, they use the Linux syscall on the locked wheels' 64-bit x86_64 and aarch64 architectures.
+Kernel errors and exit timeouts still fail acceptance. For recovery acceptance, first reproduce the stall with an isolated
+`1.4.0.post1` environment and `--scenario restart-recovery --keep-failed-storage`.
+Stop its engine before freezing the failed `tmp/pc-*/seekdb`
+directory. Keep the original immutable. This scenario creates synthetic data in the `powercontext` database, including
+the `preserved` content and `Keep` tag expected by the fixture test. A healthy database or a fixture with different
+schema or sentinels cannot establish recovery of this defect. Compare the old and fixed engines on separate copies of
+the same frozen fault fixture:
+
+```bash
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \
+  --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture
+```
+
+The fixture gate opens only a disposable copy, creates and commits the baseline revision, verifies the original data,
+reopens to verify it and commit a new write, then reopens once more to read that write. Preserve the fixture and the
+old-engine failure evidence locally; CI uploads diagnostic files, not physical databases. Do not infer upgrade support
+from these gates: the full installed-release upgrade matrix remains required.
+
 ## Backup choices
 
 | Policy | Behavior | Options |

docs/zh/docs/operate/database-migrations.md

@@ -337,6 +337,47 @@ powercontext server db-migrate verify --env-file oceanbase.env \
 
 使用最新计划中的 `plan_id` 和 `coordination.pending_run_id`。重新声明既有手动备份，不用部分状态的新备份替换原恢复点。正常升级仍只确认一次；中断恢复另需用户显式作出 DBA 声明，PC 展示计划时不会自动替用户确认。
 
+## 原生 SeekDB 重启与恢复验收
+
+嵌入式引擎必须使用 `pylibseekdb>=1.4.0.post2,<2`，事务恢复修复由该原生依赖提供。迁移检查保留
+`SHOW TRIGGERS`，并在操作之间释放最后一个客户端。分别在 Python 3.11 和 Python 3.14 环境中，从仓库根目录执行：
+
+```bash
+uv sync --locked --extra seekdb
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart
+```
+
+每次运行使用新的输出目录。migration 场景选择原 issue 的
+`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` 节点，保留自然客户端生命周期。释放客户端不代表
+原生进程已经退出，紧接着打开可能复用该进程。restart-recovery 场景检查元数据查询、提交、确认最后一个客户端关闭后
+进程退出、基线 DDL，以及重开后内容和标签的保留；即使不通过探针调用，该测试也会等待内核进程退出通知。
+每次必须恰好一个测试通过，跳过、报错和超时均不算通过。`--restart` 在每次关闭时最多等待 10 秒确认原生进程退出，
+再要求各代 PID 不同，且下一次打开之前存在退出记录。等待超时直接失败，不自动重试。
+证据包括 Python 和绑定版本、原生二进制 SHA-256、实例生命周期、
+SQL 日志、压缩后的原生日志，以及慢执行时的事务快照。独立的延长观察工作流仅用于诊断，不能用它的较长时限替代正常验收时限。
+
+两个显式重开验收均在 Linux 上执行，每次重开前通过内核退出事件确认上一代进程已结束，不使用固定启动等待。
+测试优先使用 Python 的 `os.pidfd_open`，其次使用可选的 libc 封装；两者均缺失时，仅在锁定 wheels 对应的
+64 位 x86_64 和 aarch64 架构上调用 Linux syscall。内核报错或退出等待超时仍会导致验收失败。
+故障库验收应先在隔离的 `1.4.0.post1` 环境中，
+使用 `--scenario restart-recovery --keep-failed-storage` 复现阻塞。
+停止引擎后，冻结失败输出中的 `tmp/pc-*/seekdb` 目录，并保持原件不变。此场景在 `powercontext` 数据库中创建合成数据，
+包括故障库测试要求的 `preserved` 内容和 `Keep` 标签。健康库、结构或哨兵数据不同的库，不能用于证明该缺陷已恢复。
+使用同一冻结故障库的独立副本，对比旧引擎与修复后的引擎：
+
+```bash
+uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \
+  --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \
+  --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture
+```
+
+故障库门禁只打开临时副本：创建并提交基线版本表、验证原始数据；重开后再次验证并提交新写入；第三次打开时读回该写入。
+故障库原件与旧版失败证据保存在本地；CI 只上传诊断文件，不上传物理数据库。这些门禁不代表完整升级支持，
+发布仍须通过安装产物的全部升级矩阵。
+
 ## 备份选择
 
 | 策略 | 行为 | 参数 |