diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index f9f2a53bf4..cabd12f66b 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -31,10 +31,11 @@ tests = [ ] evaluation = ["evaluation/**"] database = [ + "tests/seekdb_support.py", "tests/builtin/persistence/**", "tests/builtin/*native_code*", "tests/builtin/runtime/*recall*", "tests/builtin/runtime/test_statistics.py", "tests/builtin/runtime/test_model_usage_recorder.py", "tests/builtin/runtime/test_prepared_context.py", "tests/e2e/*atomic*", "tests/e2e/*mysql*", - "tests/e2e/*native_code*", "tests/e2e/test_recall_sufficiency_gate.py", + "tests/e2e/*native_code*", "tests/e2e/test_seekdb_restart_recovery.py", "tests/e2e/test_recall_sufficiency_gate.py", "tests/e2e/test_context_text_assembly.py", "tests/e2e/test_scope_search_migration.py", "scripts/run_atomic_memory_clean_migration.py", ".github/workflows/e2e-harness.yml", ".github/actions/setup-upgrade-oceanbase/**", @@ -114,7 +115,13 @@ release_schema = [ "tests/release_upgrades/test_full_server.py", ".github/workflows/capture-release-schema.yml", ".github/actions/setup-upgrade-oceanbase/**", ] -seekdb_lock = ["scripts/probe_seekdb_migration_lock.py", ".github/workflows/seekdb-lock-probe.yml", ".github/workflows/seekdb-lock-observation.yml"] +seekdb_lock = [ + "scripts/probe_seekdb_migration_lock.py", ".github/workflows/seekdb-lock-probe.yml", + ".github/workflows/seekdb-lock-observation.yml", "tests/seekdb_support.py", + "tests/test_seekdb_migration_probe.py", "tests/e2e/test_seekdb_restart_recovery.py", + "tests/builtin/persistence/test_mysql_migration_runner.py", + "src/powercontext/builtin/persistence/**", "pyproject.toml", "uv.lock", +] skill_guidance = [ "evaluation/skills/skill-up/**", "integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/**", diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 95679a1bbe..6acc851d96 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -477,6 +477,7 @@ jobs: fail-fast: false matrix: python-version: ["3.11", "3.14"] + migration-shard: [0, 1] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -490,9 +491,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Test code indexing with embedded seekdb + if: matrix.migration-shard == 0 run: make code-seekdb-test - name: Test database bootstrap with embedded seekdb without skips + if: matrix.migration-shard == 0 # Two 360s native-process deadlines still leave room for the remaining # bootstrap cases and diagnostic upload before the step deadline. timeout-minutes: 20 @@ -518,6 +521,7 @@ jobs: PYCODE - name: Test batched archive validation with embedded seekdb + if: matrix.migration-shard == 0 timeout-minutes: 5 run: | uv run --locked --extra seekdb python -c "import pylibseekdb" @@ -526,10 +530,11 @@ jobs: - name: Test unified migrations with embedded seekdb if: ${{ !cancelled() && steps.seekdb_setup.outcome == 'success' }} - # Full-Server cases run in separate processes and exceed fifteen minutes - # in aggregate; each runner keeps its 180-second deadline below. + # Last-client shutdown adds native startup work. Partition runner nodes; + # every node keeps its 180-second deadline. timeout-minutes: 25 env: + SEEKDB_MIGRATION_SHARD: ${{ matrix.migration-shard }} POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" PYTHONUNBUFFERED: "1" @@ -549,16 +554,18 @@ jobs: } trap collect_diagnostics EXIT df -h . /tmp - uv run --locked --extra seekdb pytest -vv \ - -o faulthandler_timeout=120 --durations=20 \ - tests/builtin/persistence/test_database_migration_backends.py \ - tests/builtin/persistence/test_database_backup_providers.py \ - tests/builtin/persistence/test_database_migration_connections.py \ - tests/e2e/test_topic_memory_recovery_migration.py \ - tests/builtin/persistence/test_mysql_migration_checks.py \ - tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ - 2>&1 | tee seekdb-migrations.log - uv run --locked --extra seekdb pytest --collect-only -q --color=no \ + if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then + uv run --locked --extra seekdb --extra code pytest -vv \ + -o faulthandler_timeout=120 --durations=20 \ + tests/builtin/persistence/test_database_migration_backends.py \ + tests/builtin/persistence/test_database_backup_providers.py \ + tests/builtin/persistence/test_database_migration_connections.py \ + tests/e2e/test_topic_memory_recovery_migration.py \ + tests/builtin/persistence/test_mysql_migration_checks.py \ + tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ + 2>&1 | tee seekdb-migrations.log + fi + uv run --locked --extra seekdb --extra code pytest --collect-only -q --color=no \ tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ 2>&1 | tee seekdb-runner-collection.log | tee -a seekdb-migrations.log mapfile -t seekdb_runner_nodes < <( @@ -569,25 +576,31 @@ jobs: echo "No seekdb migration runner tests were collected" >&2 exit 1 fi - for seekdb_runner_node in "${seekdb_runner_nodes[@]}"; do - uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \ + for index in "${!seekdb_runner_nodes[@]}"; do + if [ "$((index % 2))" -ne "$SEEKDB_MIGRATION_SHARD" ]; then + continue + fi + seekdb_runner_node="${seekdb_runner_nodes[$index]}" + uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \ "$seekdb_runner_node" \ 2>&1 | tee -a seekdb-migrations.log done # These are required independent runs, not retries: any failure stops # the job. Exercise the lifecycle race on both supported matrix ends. - for repetition in 2 3; do - echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log - uv run --locked --extra seekdb python .github/scripts/run_seekdb_migration_test.py \ - 'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \ - 2>&1 | tee -a seekdb-migrations.log - done + if [ "$SEEKDB_MIGRATION_SHARD" -eq 0 ]; then + for repetition in 2 3; do + echo "Baseline adoption stability run $repetition/3" | tee -a seekdb-migrations.log + uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \ + 'tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]' \ + 2>&1 | tee -a seekdb-migrations.log + done + fi - name: Upload seekdb bootstrap diagnostics if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-bootstrap-${{ matrix.python-version }} + name: seekdb-bootstrap-${{ matrix.python-version }}-${{ matrix.migration-shard }} path: .powercontext/database-bootstrap-ci/ if-no-files-found: ignore @@ -595,7 +608,7 @@ jobs: if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-migrations-${{ matrix.python-version }} + name: seekdb-migrations-${{ matrix.python-version }}-${{ matrix.migration-shard }} path: | seekdb-migrations.log seekdb-native.log diff --git a/.github/workflows/seekdb-lock-probe.yml b/.github/workflows/seekdb-lock-probe.yml index 7d5a4ebc70..a961dc86a4 100644 --- a/.github/workflows/seekdb-lock-probe.yml +++ b/.github/workflows/seekdb-lock-probe.yml @@ -28,22 +28,41 @@ jobs: needs: changes if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').seekdb_lock) }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.14'] + scenario: [migration, restart-recovery] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - uses: ./.github/actions/setup-python-env with: - python-version: '3.11' + python-version: ${{ matrix.python }} - name: Reproduce migration lifecycle with complete native evidence - run: uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py --attempts 12 + env: + SEEKDB_PROBE_SCENARIO: ${{ matrix.scenario }} + run: | + restart_args=() + if [ "$SEEKDB_PROBE_SCENARIO" = restart-recovery ]; then + restart_args+=(--restart) + fi + uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario "$SEEKDB_PROBE_SCENARIO" "${restart_args[@]}" --attempts 12 --timeout 180 - name: Upload native logs, lifecycle and live transaction snapshots if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: seekdb-lock-probe-${{ github.sha }} - path: .powercontext/seekdb-lock-probe + name: seekdb-lock-probe-${{ matrix.python }}-${{ matrix.scenario }}-${{ github.sha }} + path: | + .powercontext/seekdb-lock-probe/environment.json + .powercontext/seekdb-lock-probe/attempt-*/*.json + .powercontext/seekdb-lock-probe/attempt-*/*.jsonl + .powercontext/seekdb-lock-probe/attempt-*/*.xml + .powercontext/seekdb-lock-probe/attempt-*/*.log + .powercontext/seekdb-lock-probe/attempt-*/*.gz include-hidden-files: true if-no-files-found: error retention-days: 7 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 95e76d08de..2ad44358c9 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -435,6 +435,53 @@ Use the latest plan's `plan_id` and `coordination.pending_run_id`. Reconfirm the the original recovery point with a backup of partial state. Normal upgrades still use one confirmation. Recovery requires the additional explicit DBA declaration; PC never invents it when showing the plan. +## Native SeekDB restart and recovery acceptance + +The embedded engine must use `pylibseekdb>=1.4.0.post2,<2`. The native transaction recovery fix is supplied by +that dependency. The migration checks retain `SHOW TRIGGERS` and release the last client between operations. +Run the following from the repository root, separately under Python 3.11 and Python 3.14: + +```bash +uv sync --locked --extra seekdb +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart +``` + +Use a new output directory for each run. The migration scenario selects +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` with its natural client lifecycle. Releasing a +client does not imply that the native process has already exited; a rapid reopen may reuse it. The restart scenario +checks metadata inspection, commit, confirmed last-client shutdown, baseline DDL and preserved content and tags after +reopening. Its test waits for kernel process-exit notification even when invoked without the probe. Each attempt must +report exactly one passing test, with no skip, error or timeout. `--restart` explicitly waits up to 10 seconds for native +exit on each close, then requires distinct process IDs and an exit record before each subsequent open. A timeout fails +the attempt; it does not retry. Evidence includes the Python and binding versions, native binary SHA-256, instance +lifetimes, SQL trace, compressed native logs and live transaction snapshots for slow attempts. The separate extended +observation workflow is diagnostic; its longer deadline does not replace the ordinary acceptance deadline. + +Both explicit restart gates run on Linux and observe kernel process-exit notifications before each reopen; +neither uses a fixed startup delay. The tests prefer Python's `os.pidfd_open`, then the optional libc wrapper. +When both are absent, they use the Linux syscall on the locked wheels' 64-bit x86_64 and aarch64 architectures. +Kernel errors and exit timeouts still fail acceptance. For recovery acceptance, first reproduce the stall with an isolated +`1.4.0.post1` environment and `--scenario restart-recovery --keep-failed-storage`. +Stop its engine before freezing the failed `tmp/pc-*/seekdb` +directory. Keep the original immutable. This scenario creates synthetic data in the `powercontext` database, including +the `preserved` content and `Keep` tag expected by the fixture test. A healthy database or a fixture with different +schema or sentinels cannot establish recovery of this defect. Compare the old and fixed engines on separate copies of +the same frozen fault fixture: + +```bash +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \ + --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture +``` + +The fixture gate opens only a disposable copy, creates and commits the baseline revision, verifies the original data, +reopens to verify it and commit a new write, then reopens once more to read that write. Preserve the fixture and the +old-engine failure evidence locally; CI uploads diagnostic files, not physical databases. Do not infer upgrade support +from these gates: the full installed-release upgrade matrix remains required. + ## Backup choices | Policy | Behavior | Options | diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index bf6b9ead5f..0f43258f79 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -337,6 +337,47 @@ powercontext server db-migrate verify --env-file oceanbase.env \ 使用最新计划中的 `plan_id` 和 `coordination.pending_run_id`。重新声明既有手动备份,不用部分状态的新备份替换原恢复点。正常升级仍只确认一次;中断恢复另需用户显式作出 DBA 声明,PC 展示计划时不会自动替用户确认。 +## 原生 SeekDB 重启与恢复验收 + +嵌入式引擎必须使用 `pylibseekdb>=1.4.0.post2,<2`,事务恢复修复由该原生依赖提供。迁移检查保留 +`SHOW TRIGGERS`,并在操作之间释放最后一个客户端。分别在 Python 3.11 和 Python 3.14 环境中,从仓库根目录执行: + +```bash +uv sync --locked --extra seekdb +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario migration --attempts 12 --timeout 180 --output .powercontext/seekdb-migration +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario restart-recovery --restart --attempts 12 --timeout 180 --output .powercontext/seekdb-restart +``` + +每次运行使用新的输出目录。migration 场景选择原 issue 的 +`test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]` 节点,保留自然客户端生命周期。释放客户端不代表 +原生进程已经退出,紧接着打开可能复用该进程。restart-recovery 场景检查元数据查询、提交、确认最后一个客户端关闭后 +进程退出、基线 DDL,以及重开后内容和标签的保留;即使不通过探针调用,该测试也会等待内核进程退出通知。 +每次必须恰好一个测试通过,跳过、报错和超时均不算通过。`--restart` 在每次关闭时最多等待 10 秒确认原生进程退出, +再要求各代 PID 不同,且下一次打开之前存在退出记录。等待超时直接失败,不自动重试。 +证据包括 Python 和绑定版本、原生二进制 SHA-256、实例生命周期、 +SQL 日志、压缩后的原生日志,以及慢执行时的事务快照。独立的延长观察工作流仅用于诊断,不能用它的较长时限替代正常验收时限。 + +两个显式重开验收均在 Linux 上执行,每次重开前通过内核退出事件确认上一代进程已结束,不使用固定启动等待。 +测试优先使用 Python 的 `os.pidfd_open`,其次使用可选的 libc 封装;两者均缺失时,仅在锁定 wheels 对应的 +64 位 x86_64 和 aarch64 架构上调用 Linux syscall。内核报错或退出等待超时仍会导致验收失败。 +故障库验收应先在隔离的 `1.4.0.post1` 环境中, +使用 `--scenario restart-recovery --keep-failed-storage` 复现阻塞。 +停止引擎后,冻结失败输出中的 `tmp/pc-*/seekdb` 目录,并保持原件不变。此场景在 `powercontext` 数据库中创建合成数据, +包括故障库测试要求的 `preserved` 内容和 `Keep` 标签。健康库、结构或哨兵数据不同的库,不能用于证明该缺陷已恢复。 +使用同一冻结故障库的独立副本,对比旧引擎与修复后的引擎: + +```bash +uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py \ + --scenario fixture-recovery --fixture /absolute/path/to/frozen/seekdb \ + --restart --attempts 1 --timeout 75 --output .powercontext/seekdb-fixture +``` + +故障库门禁只打开临时副本:创建并提交基线版本表、验证原始数据;重开后再次验证并提交新写入;第三次打开时读回该写入。 +故障库原件与旧版失败证据保存在本地;CI 只上传诊断文件,不上传物理数据库。这些门禁不代表完整升级支持, +发布仍须通过安装产物的全部升级矩阵。 + ## 备份选择 | 策略 | 行为 | 参数 | diff --git a/scripts/probe_seekdb_migration_lock.py b/scripts/probe_seekdb_migration_lock.py index 9839491bce..31057ac59e 100644 --- a/scripts/probe_seekdb_migration_lock.py +++ b/scripts/probe_seekdb_migration_lock.py @@ -18,14 +18,18 @@ import argparse import gzip +import hashlib import json import os import platform import shutil +import signal import subprocess import sys import time +from contextlib import suppress from datetime import UTC, datetime +from importlib import import_module from importlib.metadata import version from pathlib import Path from types import ModuleType @@ -36,46 +40,70 @@ NODE = ( "tests/builtin/persistence/test_mysql_migration_runner.py::" - "test_mysql_baseline_adoption_recovers_an_empty_committed_version_table[seekdb]" + "test_mysql_committed_revision_recovers_before_evidence_advance[seekdb]" ) +SCENARIOS = { + "migration": NODE, + "restart-recovery": ( + "tests/e2e/test_seekdb_restart_recovery.py::test_metadata_inspection_survives_last_client_restarts" + ), + "fixture-recovery": ( + "tests/e2e/test_seekdb_restart_recovery.py::test_saved_fault_recovers_and_remains_usable_after_reopen" + ), +} + def record(directory: Path, event: str, **values: object) -> None: with (directory / "lifecycle.jsonl").open("a") as output: - output.write(json.dumps({"time": datetime.now(UTC).isoformat(), "event": event, **values}) + "\n") + output.write( + json.dumps({"time": datetime.now(UTC).isoformat(), "event": event, **values}, ensure_ascii=False) + "\n" + ) def pytest_sessionstart(session: pytest.Session) -> None: """Optional pytest plugin: observe only this subprocess's native instance lifetime.""" from powercontext.builtin.persistence.seekdb import profile + from tests.seekdb_support import RestartingInstance directory = Path(os.environ["POWERCONTEXT_SEEKDB_PROBE"]) + restart = os.environ.get("POWERCONTEXT_SEEKDB_REQUIRE_RESTART") == "1" original = profile._open_instance class ObservedInstance: - def __init__(self, instance: Any, path: Path) -> None: + def __init__(self, instance: Any, path: Path, native_pid: int) -> None: self.instance = instance self.path = path + self.native_pid = native_pid def connection_options(self) -> Any: return self.instance.connection_options() def close(self) -> None: - record(directory, "closing", path=str(self.path)) + record(directory, "closing", path=str(self.path), native_pid=self.native_pid) try: self.instance.close() + if restart: + record(directory, "native_exited", path=str(self.path), native_pid=self.native_pid) finally: - record(directory, "close_returned", path=str(self.path)) + record(directory, "close_returned", path=str(self.path), native_pid=self.native_pid) async def observed_open(module: ModuleType, path: Path) -> profile._SeekDBInstance: record(directory, "opening", path=str(path)) instance = await original(module, path) + try: + native_pid = int((path / "run/seekdb.pid").read_text().strip()) + except (OSError, ValueError): + instance.close() + raise + if restart: + instance = RestartingInstance(instance, native_pid) options = dict(instance.connection_options()) # Embedded scratch instance only. Never capture credentials or external connection settings. socket = options.get("unix_socket") (directory / "socket.json").write_text(json.dumps({"unix_socket": socket})) - record(directory, "opened", path=str(path)) - return ObservedInstance(instance, path) + record(directory, "opened", path=str(path), native_pid=native_pid) + return ObservedInstance(instance, path, native_pid) patch = pytest.MonkeyPatch() patch.setattr(profile, "_open_instance", observed_open) @@ -171,7 +199,50 @@ def successful_report(report: Path, code: int) -> bool: ) -def run_attempt(directory: Path, timeout: int) -> bool: +def stop_process(process: subprocess.Popen[bytes]) -> None: + """Bound shutdown of a probe child, including an unresponsive native client.""" + if process.poll() is None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def restart_evidence(directory: Path) -> dict[str, Any]: + """Require confirmed exit before each subsequent native generation opens.""" + lifecycle = directory / "lifecycle.jsonl" + events = [json.loads(line) for line in lifecycle.read_text().splitlines()] if lifecycle.exists() else [] + processes: dict[str, list[int]] = {} + running: dict[str, int] = {} + valid = True + for event in events: + if event["event"] == "opened": + path, pid = event["path"], event["native_pid"] + pids = processes.setdefault(path, []) + valid = valid and path not in running and pid > 0 and pid not in pids + pids.append(pid) + running[path] = pid + elif event["event"] == "native_exited": + path, pid = event["path"], event["native_pid"] + valid = valid and running.get(path) == pid + running.pop(path, None) + passed = valid and not running and any(len(pids) > 1 for pids in processes.values()) + return {"passed": passed, "processes": processes} + + +def run_attempt( + directory: Path, + timeout: int, + *, + node: str = NODE, + restart: bool = False, + fixture: Path | None = None, + keep_failed_storage: bool = False, +) -> bool: directory.mkdir(parents=True) temporary = directory / "tmp" temporary.mkdir() @@ -179,8 +250,12 @@ def run_attempt(directory: Path, timeout: int) -> bool: os.environ, TMPDIR=str(temporary), POWERCONTEXT_SEEKDB_PROBE=str(directory), + POWERCONTEXT_SEEKDB_REQUIRE_RESTART=str(int(restart)), POWERCONTEXT_TEST_MIGRATION_SEEKDB="1", POWERCONTEXT_TEST_MIGRATION_SQL_LOG="1", + POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY="1", + POWERCONTEXT_SEEKDB_RECOVERY_FIXTURE=str(fixture or ""), + POWERCONTEXT_SEEKDB_KEEP_FAILED_STORAGE=str(int(keep_failed_storage)), ) report = directory / "junit.xml" command = [ @@ -194,9 +269,9 @@ def run_attempt(directory: Path, timeout: int) -> bool: "-vv", "--log-cli-level=INFO", "-o", - "faulthandler_timeout=120", + f"faulthandler_timeout={min(120, max(1, timeout // 2))}", f"--junitxml={report}", - NODE, + node, ] logs = NativeLogs(temporary, directory) snapshots: list[tuple[subprocess.Popen[bytes], Any, float]] = [] @@ -204,16 +279,24 @@ def run_attempt(directory: Path, timeout: int) -> bool: next_snapshot = 45.0 timed_out = False process = None + code = -1 + passed = False + restarts = None try: with (directory / "pytest.log").open("w") as output: - process = subprocess.Popen(command, env=environment, stdout=output, stderr=subprocess.STDOUT) # noqa: S603 + process = subprocess.Popen( # noqa: S603 + command, env=environment, stdout=output, stderr=subprocess.STDOUT, start_new_session=True + ) while process.poll() is None: logs.poll() elapsed = time.monotonic() - started if elapsed >= next_snapshot: log = (directory / f"snapshot-{int(next_snapshot)}.jsonl").open("w") probe = subprocess.Popen( # noqa: S603 - [sys.executable, __file__, "--snapshot", str(directory)], stdout=log, stderr=subprocess.STDOUT + [sys.executable, __file__, "--snapshot", str(directory)], + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, ) snapshots.append((probe, log, time.monotonic())) next_snapshot += 45 @@ -222,60 +305,126 @@ def run_attempt(directory: Path, timeout: int) -> bool: probe.kill() if elapsed >= timeout: timed_out = True - process.terminate() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() + stop_process(process) break time.sleep(0.1) code = process.wait() + passed = not timed_out and successful_report(report, code) + if restart and passed: + restarts = restart_evidence(directory) + passed = restarts["passed"] finally: - if process is not None and process.poll() is None: - process.kill() - process.wait() + if process is not None: + stop_process(process) for probe, output, _ in snapshots: - if probe.poll() is None: - probe.kill() - probe.wait() + stop_process(probe) output.close() logs.close() # Only remove this attempt's disposable engine files, after preserving native logs. - shutil.rmtree(temporary) - passed = not timed_out and successful_report(report, code) - result = {"passed": passed, "exit_code": code, "timed_out": timed_out, "seconds": time.monotonic() - started} + if not keep_failed_storage or passed: + shutil.rmtree(temporary) + result = { + "passed": passed, + "exit_code": code, + "timed_out": timed_out, + "seconds": time.monotonic() - started, + "restarts": restarts, + } (directory / "result.json").write_text(json.dumps(result, indent=2) + "\n") print(directory.name, json.dumps(result), flush=True) return passed -def main() -> int: +def positive_integer(value: str) -> int: + parsed = int(value) + if parsed < 1: + message = "must be positive" + raise argparse.ArgumentTypeError(message) + return parsed + + +def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--snapshot", type=Path) parser.add_argument("--output", type=Path, default=Path(".powercontext/seekdb-lock-probe")) - parser.add_argument("--attempts", type=int, default=12) - parser.add_argument("--timeout", type=int, default=180) + parser.add_argument("--attempts", type=positive_integer, default=12) + parser.add_argument("--node", help="Exact pytest node, overriding the selected scenario.") + parser.add_argument("--scenario", choices=SCENARIOS, default="migration") + parser.add_argument( + "--fixture", + type=Path, + help="Frozen synthetic baseline with the recovery test's sentinels; copied before opening.", + ) + parser.add_argument( + "--keep-failed-storage", action="store_true", help="Retain failed scratch storage for recovery acceptance." + ) + parser.add_argument( + "--restart", action="store_true", help="Wait for Linux native process exit before each reopen and verify it." + ) + parser.add_argument("--timeout", type=positive_integer, default=180) arguments = parser.parse_args() + if arguments.snapshot: + return arguments + if arguments.scenario == "fixture-recovery" and arguments.fixture is None: + parser.error("--scenario fixture-recovery requires --fixture") + if arguments.fixture is not None: + arguments.fixture = arguments.fixture.resolve() + if not (arguments.fixture / "etc/seekdb.data_version.bin").is_file(): + parser.error("--fixture must be a recognized native seekdb directory") + node = arguments.node or SCENARIOS[arguments.scenario] + directory = arguments.output.resolve() + if arguments.fixture is not None and directory.is_relative_to(arguments.fixture): + parser.error("--output must be outside the immutable --fixture directory") + if directory.exists(): + parser.error("--output must be a new directory to preserve previous evidence") + arguments.output = directory + arguments.node = node + return arguments + + +def main() -> int: + arguments = parse_arguments() if arguments.snapshot: snapshot(arguments.snapshot) return 0 - directory = arguments.output.resolve() - directory.mkdir(parents=True, exist_ok=True) + directory = arguments.output + node = arguments.node + directory.mkdir(parents=True) + binding = import_module("pylibseekdb") + binding_file = binding.__file__ + if binding_file is None: + message = "The SeekDB binding has no module file; cannot locate its native engine" + raise RuntimeError(message) + native_engine = Path(binding_file).with_name("seekdb") + with native_engine.open("rb") as engine_file: + native_digest = hashlib.file_digest(engine_file, "sha256").hexdigest() (directory / "environment.json").write_text( json.dumps( { "platform": platform.platform(), "python": sys.version, "pylibseekdb": version("pylibseekdb"), - "node": NODE, + "seekdb_engine": str(native_engine), + "seekdb_engine_sha256": native_digest, + "node": node, + "scenario": arguments.scenario, + "restart": arguments.restart, }, indent=2, + ensure_ascii=False, ) + "\n" ) for attempt in range(1, arguments.attempts + 1): attempt_directory = directory / f"attempt-{attempt:02d}" - if not run_attempt(attempt_directory, arguments.timeout): + if not run_attempt( + attempt_directory, + arguments.timeout, + node=node, + restart=arguments.restart, + fixture=arguments.fixture, + keep_failed_storage=arguments.keep_failed_storage, + ): return 1 # Keep a complete successful baseline and every failure. Other successful # repeats retain their Python trace, lifecycle and test result only. diff --git a/tests/builtin/persistence/test_mysql_migration_checks.py b/tests/builtin/persistence/test_mysql_migration_checks.py index c0e4a7d6ce..cb50afac8e 100644 --- a/tests/builtin/persistence/test_mysql_migration_checks.py +++ b/tests/builtin/persistence/test_mysql_migration_checks.py @@ -30,7 +30,6 @@ from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, _expression_sql, mysql_inventory from powercontext.builtin.persistence.seekdb import SeekDBConfig -from tests.builtin.persistence.test_mysql_migration_runner import _seekdb_scenario_lifetime from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase _FIXTURE = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json" @@ -179,9 +178,7 @@ def reflect(connection, _identity, guard): [("a % 2 = 0", "INTEGER", 2, 3), ("a = '100%s%%'", "VARCHAR(32)", "100%s%%", "different")], ids=["remainder", "percent_literal"], ) -def test_real_seekdb_percent_checks_validate_data_and_plan( - short_tmp_path, monkeypatch, expression, column_type, valid, invalid -): +def test_real_seekdb_percent_checks_validate_data_and_plan(short_tmp_path, expression, column_type, valid, invalid): pytest.importorskip("pylibseekdb") # Extend an isolated bundle, preserving the shipped layouts and their eligibility. directory = short_tmp_path / "bundle" @@ -222,23 +219,22 @@ def invalidate(connection, _identity, guard): with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): schema.validate_data(connection, "p0001") - with _seekdb_scenario_lifetime(monkeypatch): - fresh = MySQLMigrationRunner( - MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "fresh_seekdb")), - bundle, - evidence_directory=short_tmp_path / "fresh_evidence", - ) - fresh_plan = fresh.plan(backup_policy="skip") - fresh.apply( - plan_id=fresh_plan.plan_id, - accepted=True, - maintenance_confirmed=True, - backup_policy="skip", - accept_no_backup=True, - ) - assert fresh.verify().state == "ready" - connections.run(prepare, writable=True) - assert runner.plan().state == "migration_required" - connections.run(invalidate, writable=True) - with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): - runner.plan() + fresh = MySQLMigrationRunner( + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "fresh_seekdb")), + bundle, + evidence_directory=short_tmp_path / "fresh_evidence", + ) + fresh_plan = fresh.plan(backup_policy="skip") + fresh.apply( + plan_id=fresh_plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="skip", + accept_no_backup=True, + ) + assert fresh.verify().state == "ready" + connections.run(prepare, writable=True) + assert runner.plan().state == "migration_required" + connections.run(invalidate, writable=True) + with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): + runner.plan() diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index f50ab3d23b..90fe367760 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -27,6 +27,7 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, datetime +from importlib import import_module from pathlib import Path from typing import Literal, cast from unittest.mock import Mock @@ -126,44 +127,19 @@ async def manage(*, create: bool) -> None: pytest.fail("OceanBase owned test database cleanup failed", pytrace=False) -@contextmanager -def _seekdb_scenario_lifetime(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: - binding = pytest.importorskip("pylibseekdb") - open_instance = binding.aopen - instances = {} - - async def open_with_scenario_lifetime(path: str): - # Keep a real native lease across this scenario's independent - # maintenance connections. Rapid last-client exits in seekdb 1.4 - # can strand a background DDL lock on restart. Engine restart and - # cross-process ownership have separate connection acceptance tests. - # Acquire lazily so inspecting a missing target still creates nothing. - if path not in instances: - instances[path] = await open_instance(path) - return await open_instance(path) - - monkeypatch.setattr(binding, "aopen", open_with_scenario_lifetime) - try: - yield - finally: - for instance in instances.values(): - instance.close() - - @pytest.fixture(params=("seekdb", "oceanbase"), ids=("seekdb", "oceanbase")) def backend( request: pytest.FixtureRequest, short_tmp_path: Path, caplog: pytest.LogCaptureFixture, - monkeypatch: pytest.MonkeyPatch, ) -> Iterator[_BackendTarget]: if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SQL_LOG") == "1": caplog.set_level(logging.INFO, logger="sqlalchemy.engine") if request.param == "seekdb": if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1": pytest.skip("real seekdb probe not enabled") - with _seekdb_scenario_lifetime(monkeypatch): - yield _BackendTarget(SeekDBConfig(path=short_tmp_path / "seekdb"), short_tmp_path / "evidence") + import_module("pylibseekdb") + yield _BackendTarget(SeekDBConfig(path=short_tmp_path / "seekdb"), short_tmp_path / "evidence") return value = os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL") if not value: @@ -1218,9 +1194,24 @@ def interrupt(journal, evidence) -> None: recovery = replacement.plan(backup_policy="manual") assert recovery.source_revision == "p0002" replacement = _acknowledge_known_completed_interruption(backend, recovery) - assert _apply(replacement, policy="manual").state == "ready" + upgrade = replacement._upgrade + recovered_p0003 = False + + def observe_recovery(connection, config, source, target, guard) -> None: + nonlocal recovered_p0003 + upgrade(connection, config, source, target, guard) + if target == "p0003": + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one() == "p0003" + recovered_p0003 = True + + with monkeypatch.context() as patch: + patch.setattr(replacement, "_upgrade", observe_recovery) + assert _apply(replacement, policy="manual").state == "ready" + assert recovered_p0003, "Recovery did not reach p0003" assert replacement.verify().revision == "p0004" - assert _read_state(backend)["content"] == [(b'{"content":"preserved"}',)] + state = _read_state(backend) + assert state["content"] == [(b'{"content":"preserved"}',)] + assert state["tags"] == [("Keep",)] def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledgement(backend: _BackendTarget) -> None: diff --git a/tests/e2e/test_seekdb_restart_recovery.py b/tests/e2e/test_seekdb_restart_recovery.py new file mode 100644 index 0000000000..7a3baa3038 --- /dev/null +++ b/tests/e2e/test_seekdb_restart_recovery.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native restart/recovery release gates; no additional instance lease is held.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import tempfile +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from importlib import import_module +from pathlib import Path + +import pytest +from sqlalchemy import Connection, text + +from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections +from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.seekdb import SeekDBConfig, profile +from tests.seekdb_support import RestartingInstance + +pytestmark = pytest.mark.skipif( + os.environ.get("POWERCONTEXT_TEST_SEEKDB_RESTART_RECOVERY") != "1", + reason="native seekdb restart/recovery release gate not enabled", +) + + +@pytest.fixture(autouse=True) +def native_restarts(monkeypatch: pytest.MonkeyPatch) -> None: + original = profile._open_instance + + async def open_generation(binding, path: Path): + instance = await original(binding, path) + try: + pid = int((path / "run/seekdb.pid").read_text().strip()) + except (OSError, ValueError): + instance.close() + raise + return RestartingInstance(instance, pid) + + monkeypatch.setattr(profile, "_open_instance", open_generation) + + +@pytest.fixture +def recovery_path() -> Iterator[Path]: + path = Path(tempfile.mkdtemp(prefix="pc-")) + try: + yield path + finally: + # The probe owns the parent directory and removes successful attempts. + # Keep failures even when the client timeout lets pytest exit normally. + if os.environ.get("POWERCONTEXT_SEEKDB_KEEP_FAILED_STORAGE") != "1": + shutil.rmtree(path) + + +def test_metadata_inspection_survives_last_client_restarts(recovery_path: Path) -> None: + import_module("pylibseekdb") + connections = MaintenanceConnections(SeekDBConfig(path=recovery_path / "seekdb")) + bundle = production_bundle() + schema = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text()) + + def seed(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + for statement in schema["mysql"]: + connection.exec_driver_sql(statement) + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('s','memory','m',1,:content)" + ), + {"content": b'{"content":"preserved"}'}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','memory','m',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_tags " + "(scope_id,family,artifact_id,target_type,target_id,tag_key_hash,tag_key,tag,assigned_at) " + "VALUES ('s','memory','m','artifact','m',UNHEX(REPEAT('00',32)),'keep','Keep','2026-09-28 12:00:00')" + ) + connection.commit() + + def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SHOW TRIGGERS").all() == [] + connection.commit() + + def adopt(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + inspect(connection, _identity, _guard) + connection.exec_driver_sql("START TRANSACTION") + connection.exec_driver_sql( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ) + connection.exec_driver_sql("INSERT INTO pc_schema_revision VALUES ('p0001')") + connection.commit() + + def verify(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").all() == [("p0001",)] + assert connection.exec_driver_sql("SELECT content FROM pc_artifacts").all() == [(b'{"content":"preserved"}',)] + assert connection.exec_driver_sql("SELECT tag FROM pc_artifact_tags").all() == [("Keep",)] + + connections.run(seed, writable=True) + connections.run(inspect) + connections.run(inspect) + connections.run(adopt, writable=True) + connections.run(verify) + + +@contextmanager +def _native_generation(binding, path: Path): + instance = asyncio.run(profile._open_instance(binding, path)) + try: + yield instance + finally: + instance.close() + + +def test_saved_fault_recovers_and_remains_usable_after_reopen(recovery_path: Path) -> None: + binding = import_module("pylibseekdb") + pymysql = import_module("pymysql") + configured = os.environ.get("POWERCONTEXT_SEEKDB_RECOVERY_FIXTURE") + if not configured: + pytest.fail("enabled recovery acceptance requires an immutable native fault fixture directory", pytrace=False) + source = Path(configured).resolve() + assert (source / "etc/seekdb.data_version.bin").is_file(), "unrecognized native fault fixture" + assert (source / "store/sstable/meta.db").is_file(), "unrecognized native fault fixture" + path = recovery_path / "seekdb" + # Open only the disposable clone, including for startup/recovery writes. + shutil.copytree( + source, + path, + ignore=lambda directory, _names: ["run"] if Path(directory) == source else [], + ) + + def verify(connection) -> None: + with connection.cursor() as cursor: + cursor.execute("SELECT version_num FROM pc_schema_revision") + assert cursor.fetchall() == (("p0001",),) + cursor.execute("SELECT content FROM pc_artifacts") + assert cursor.fetchall() == ((b'{"content":"preserved"}',),) + cursor.execute("SELECT tag FROM pc_artifact_tags") + assert cursor.fetchall() == (("Keep",),) + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + with connection.cursor() as cursor: + cursor.execute( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ) + cursor.execute("INSERT INTO pc_schema_revision VALUES ('p0001')") + connection.commit() + verify(connection) + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + verify(connection) + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE pc_recovery_write_check (id INT PRIMARY KEY)") + cursor.execute("INSERT INTO pc_recovery_write_check VALUES (1)") + connection.commit() + + with ( + _native_generation(binding, path) as instance, + pymysql.connect(**instance.connection_options(), database="powercontext", read_timeout=15) as connection, + ): + verify(connection) + with connection.cursor() as cursor: + cursor.execute("SELECT id FROM pc_recovery_write_check") + assert cursor.fetchall() == ((1,),) diff --git a/tests/e2e/test_topic_memory_recovery_migration.py b/tests/e2e/test_topic_memory_recovery_migration.py index c0d84cc18b..d463043029 100644 --- a/tests/e2e/test_topic_memory_recovery_migration.py +++ b/tests/e2e/test_topic_memory_recovery_migration.py @@ -22,7 +22,7 @@ import os import shutil import sqlite3 -from contextlib import closing, nullcontext +from contextlib import closing import pytest from sqlalchemy import Engine, event @@ -41,7 +41,6 @@ from powercontext.builtin.runtime.topic_memory_recovery import TopicMemoryRecoveryService from powercontext.cli.app import create_cli from powercontext.server.cli import app -from tests.builtin.persistence.test_mysql_migration_runner import _seekdb_scenario_lifetime from tests.builtin.runtime.test_topic_memory_processing import _repositories from tests.builtin.runtime.test_topic_memory_scope import BINDING, _capture from tests.builtin.runtime.test_topic_memory_terminal_frontier import _budget, _seed @@ -587,7 +586,7 @@ async def reopen(): @pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) @pytest.mark.parametrize("interrupted", [False, True]) def test_real_mysql_recovery_upgrade_preserves_receipt_and_resumes_registered_ddl_prefix( - backend, interrupted, short_tmp_path, tmp_path, monkeypatch + backend, interrupted, short_tmp_path, tmp_path ): async def scenario(): async with _isolated_database(backend, short_tmp_path) as (config, open_profile): @@ -661,8 +660,4 @@ def stop_during_ddl(_connection, _cursor, statement, _parameters, _context, _exe ) ).one() == (3, 9, 1000) - # Exercise committed DDL recovery across independent connections without - # triggering seekdb 1.4's rapid last-client-close runtime lock stall. - # Native restart/exclusion acceptance remains in the connection tests. - with _seekdb_scenario_lifetime(monkeypatch) if backend == "seekdb" else nullcontext(): - asyncio.run(scenario()) + asyncio.run(scenario()) diff --git a/tests/seekdb_support.py b/tests/seekdb_support.py new file mode 100644 index 0000000000..0b95340443 --- /dev/null +++ b/tests/seekdb_support.py @@ -0,0 +1,92 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Linux process-exit ownership for native SeekDB acceptance tests.""" + +from __future__ import annotations + +import ctypes +import os +import platform +import selectors +import sys +from typing import Any + + +def _open_pidfd(pid: int) -> int: + """Open a Linux pidfd without requiring a recent host libc or Python build.""" + if pid <= 0: + message = "native process ID must be positive" + raise ValueError(message) + if sys.platform != "linux": + message = "native restart acceptance requires Linux pidfd support" + raise NotImplementedError(message) + python_open = getattr(os, "pidfd_open", None) + if python_open is not None: + return python_open(pid, 0) + + libc = ctypes.CDLL(None, use_errno=True) + libc_open = getattr(libc, "pidfd_open", None) + if libc_open is not None: + libc_open.argtypes = (ctypes.c_int, ctypes.c_uint) + libc_open.restype = ctypes.c_int + descriptor = libc_open(pid, 0) + else: + # Linux UAPI: arch/x86/entry/syscalls/syscall_64.tbl and + # include/uapi/asm-generic/unistd.h (used by aarch64). + numbers = {"x86_64": 434, "aarch64": 434} + number = numbers.get(platform.machine()) + if number is None or ctypes.sizeof(ctypes.c_void_p) != 8: + message = "pidfd syscall fallback supports only the locked 64-bit x86_64 and aarch64 Linux wheels" + raise NotImplementedError(message) + syscall = libc.syscall + syscall.argtypes = (ctypes.c_long, ctypes.c_int, ctypes.c_uint) + syscall.restype = ctypes.c_long + descriptor = syscall(number, pid, 0) + if descriptor < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + return descriptor + + +class RestartingInstance: + """Release a client and require its native process to exit before returning.""" + + def __init__(self, instance: Any, native_pid: int, *, timeout: float = 10) -> None: + self.instance = instance + self.timeout = timeout + self.descriptor: int | None = None + try: + self.descriptor = _open_pidfd(native_pid) + except BaseException: + instance.close() + raise + + def connection_options(self) -> Any: + return self.instance.connection_options() + + def close(self) -> None: + descriptor = self.descriptor + if descriptor is None: + return + self.descriptor = None + try: + self.instance.close() + with selectors.DefaultSelector() as selector: + selector.register(descriptor, selectors.EVENT_READ) + if not selector.select(self.timeout): + message = "native process did not exit after last-client close" + raise TimeoutError(message) + finally: + os.close(descriptor) diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index 9695998396..d616600936 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -89,6 +89,13 @@ (["tests/test_ci_installation_roles.py"], {"tests", "packaging"}), (["scripts/capture_release_schema.py"], {"upgrade", "release_schema"}), (["scripts/probe_seekdb_migration_lock.py"], {"seekdb_lock"}), + (["tests/seekdb_support.py"], {"tests", "database", "seekdb_lock"}), + (["tests/test_seekdb_migration_probe.py"], {"tests", "database", "upgrade", "seekdb_lock"}), + (["tests/e2e/test_seekdb_restart_recovery.py"], {"tests", "database", "seekdb_lock"}), + ( + ["tests/builtin/persistence/test_mysql_migration_runner.py"], + {"tests", "database", "upgrade", "candidate_replay", "seekdb_lock"}, + ), (["tests/test_database_migration_cli.py"], {"tests", "database", "upgrade"}), ([".github/workflows/skill-guidance.yml"], {"skill_guidance"}), ([".licenserc.yaml"], set()), @@ -123,7 +130,15 @@ def test_shared_or_unclassified_changes_run_every_suite(path: str) -> None: assert all(select_suites(["docs/en/docs/quickstart.md", path], RULES).values()) -@pytest.mark.parametrize("path", ["src/powercontext/server/authz/owner_recovery.py", "uv.lock", "pyproject.toml"]) +@pytest.mark.parametrize( + "path", + [ + "src/powercontext/server/authz/owner_recovery.py", + "src/powercontext/builtin/persistence/mysql.py", + "uv.lock", + "pyproject.toml", + ], +) def test_shared_runtime_inputs_keep_backend_upgrade_and_consumer_coverage(path: str) -> None: selected = select_suites([path], RULES) assert { @@ -140,7 +155,7 @@ def test_shared_runtime_inputs_keep_backend_upgrade_and_consumer_coverage(path: "managed_resources", } <= {suite for suite, enabled in selected.items() if enabled} assert not selected["release_schema"] - assert not selected["seekdb_lock"] + assert selected["seekdb_lock"] is (path != "src/powercontext/server/authz/owner_recovery.py") def git(repository: Path, *arguments: str) -> str: diff --git a/tests/test_ops.py b/tests/test_ops.py index d5c6666d2e..60dcbe19cb 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -507,6 +507,7 @@ def emit(message): # owns the worker whose exit releases both locks. bootstrap = """import runpy, site, sys packages, repository, script, *arguments = sys.argv[1:] +sys.path.insert(0, packages) site.addsitedir(packages) sys.path.insert(0, repository) sys.argv = [script, *arguments] diff --git a/tests/test_seekdb_migration_probe.py b/tests/test_seekdb_migration_probe.py new file mode 100644 index 0000000000..25a75ca24e --- /dev/null +++ b/tests/test_seekdb_migration_probe.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A skipped test or retained engine must not certify native recovery.""" + +import ctypes +import errno +import json +import os +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +from scripts import probe_seekdb_migration_lock as probe +from scripts.probe_seekdb_migration_lock import restart_evidence, successful_report +from tests import seekdb_support +from tests.seekdb_support import RestartingInstance + + +@pytest.mark.parametrize("outcome", ["", "", "", ""]) +def test_native_acceptance_requires_one_passing_case(tmp_path: Path, outcome: str) -> None: + report = tmp_path / "junit.xml" + report.write_text(f"{outcome}") + assert successful_report(report, 0) is (outcome == "") + assert not successful_report(report, 1) + report.write_text("") + assert not successful_report(report, 0) + + +@pytest.mark.parametrize("pids, accepted", [([], False), ([101], False), ([101, 101], False), ([101, 102], True)]) +def test_restart_acceptance_rejects_an_engine_kept_alive(tmp_path: Path, pids: list[int], accepted: bool) -> None: + events = [ + {"event": event, "path": "/synthetic/seekdb", "native_pid": pid} + for pid in pids + for event in ("opened", "native_exited") + ] + (tmp_path / "lifecycle.jsonl").write_text("\n".join(json.dumps(event) for event in events)) + assert restart_evidence(tmp_path)["passed"] is accepted + + +@pytest.mark.parametrize("exit_before_reopen", [False, True]) +def test_different_pids_cannot_certify_restart_without_confirmed_exit(tmp_path: Path, exit_before_reopen: bool) -> None: + events = [{"event": "opened", "path": "/synthetic/seekdb", "native_pid": 101}] + exited = {"event": "native_exited", "path": "/synthetic/seekdb", "native_pid": 101} + if exit_before_reopen: + events.append(exited) + events.append({"event": "opened", "path": "/synthetic/seekdb", "native_pid": 102}) + if not exit_before_reopen: + events.append(exited) + events.append({"event": "native_exited", "path": "/synthetic/seekdb", "native_pid": 102}) + (tmp_path / "lifecycle.jsonl").write_text("\n".join(json.dumps(event) for event in events)) + assert not restart_evidence(tmp_path)["passed"] + + +def test_probe_rejects_a_binding_without_a_module_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + binding = ModuleType("pylibseekdb") + binding.__file__ = None + monkeypatch.setattr(probe, "import_module", lambda _name: binding) + monkeypatch.setattr(sys, "argv", ["probe", "--output", str(tmp_path / "probe")]) + with pytest.raises(RuntimeError, match="no module file"): + probe.main() + + +@pytest.fixture +def owned_process() -> Iterator[subprocess.Popen[bytes]]: + with subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdin.buffer.read()"], stdin=subprocess.PIPE + ) as process: + try: + yield process + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + +@pytest.fixture(params=("default", "python", "libc", "syscall")) +def pidfd_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + if request.param == "default": + return + if request.param == "python": + if not hasattr(os, "pidfd_open"): + pytest.skip("this Python build does not expose os.pidfd_open") + # A working Python API must not need any libc wrapper symbol. + monkeypatch.setattr(ctypes, "CDLL", lambda *_args, **_kwargs: SimpleNamespace()) + return + libc = ctypes.CDLL(None, use_errno=True) + monkeypatch.delattr(os, "pidfd_open", raising=False) + if request.param == "libc": + if not hasattr(libc, "pidfd_open"): + pytest.skip("this libc does not expose a pidfd_open wrapper") + else: + # Keep the real syscall entry point but hide the optional named wrapper. + monkeypatch.setattr(ctypes, "CDLL", lambda *_args, **_kwargs: SimpleNamespace(syscall=libc.syscall)) + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_last_client_close_waits_for_process_exit(owned_process: subprocess.Popen[bytes], pidfd_backend: None) -> None: + assert owned_process.stdin is not None + instance = RestartingInstance(SimpleNamespace(close=owned_process.stdin.close), owned_process.pid) + instance.close() + assert owned_process.poll() == 0 + instance.close() + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_last_client_close_rejects_a_process_that_stays_alive( + owned_process: subprocess.Popen[bytes], pidfd_backend: None +) -> None: + instance = RestartingInstance(SimpleNamespace(close=lambda: None), owned_process.pid, timeout=0) + with pytest.raises(TimeoutError, match="did not exit"): + instance.close() + assert owned_process.poll() is None + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +@pytest.mark.parametrize("error", [errno.ENOSYS, errno.EPERM]) +def test_pidfd_errors_preserve_the_failure_and_close_the_client( + owned_process: subprocess.Popen[bytes], monkeypatch: pytest.MonkeyPatch, error: int +) -> None: + def unavailable(_pid: int, _flags: int) -> int: + raise OSError(error, os.strerror(error)) + + monkeypatch.setattr(os, "pidfd_open", unavailable, raising=False) + # Falling back after a real kernel error would hide the reason for failure. + monkeypatch.setattr(ctypes, "CDLL", lambda *_args, **_kwargs: SimpleNamespace()) + assert owned_process.stdin is not None + with pytest.raises(OSError) as failed: + RestartingInstance(SimpleNamespace(close=owned_process.stdin.close), owned_process.pid) + assert failed.value.errno == error + assert owned_process.wait(timeout=10) == 0 + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_pidfd_syscall_rejects_unknown_architectures( + owned_process: subprocess.Popen[bytes], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delattr(os, "pidfd_open", raising=False) + monkeypatch.setattr(ctypes, "CDLL", lambda *_args, **_kwargs: SimpleNamespace()) + monkeypatch.setattr(seekdb_support.platform, "machine", lambda: "unknown") + assert owned_process.stdin is not None + with pytest.raises(NotImplementedError, match="locked 64-bit"): + RestartingInstance(SimpleNamespace(close=owned_process.stdin.close), owned_process.pid) + assert owned_process.wait(timeout=10) == 0 + + +@pytest.mark.skipif(sys.platform != "linux", reason="native restart acceptance uses Linux pidfd") +def test_last_client_close_supports_pidfds_above_select_limit( + owned_process: subprocess.Popen[bytes], monkeypatch: pytest.MonkeyPatch +) -> None: + import fcntl + import resource + + soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft != resource.RLIM_INFINITY and soft <= 1025: + pytest.skip("the process descriptor limit cannot exercise fd >= 1024") + original = seekdb_support._open_pidfd + + def high_descriptor(pid: int) -> int: + descriptor = original(pid) + try: + return fcntl.fcntl(descriptor, fcntl.F_DUPFD_CLOEXEC, 1024) + finally: + os.close(descriptor) + + monkeypatch.setattr(seekdb_support, "_open_pidfd", high_descriptor) + assert owned_process.stdin is not None + instance = RestartingInstance(SimpleNamespace(close=owned_process.stdin.close), owned_process.pid) + instance.close() + assert owned_process.poll() == 0 diff --git a/tox.ini b/tox.ini index 3cab314540..bccaa0da49 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,10 @@ python = [testenv] passenv = PYTHON_VERSION allowlist_externals = uv +setenv = + UV_PROJECT_ENVIRONMENT={envdir} + COVERAGE_FILE={envtmpdir}/.coverage commands = - uv sync --python {envpython} - uv run python -m pytest --doctest-modules tests --cov --cov-config=pyproject.toml --cov-report=xml - ty check + uv sync --locked --python {envpython} + uv run --no-sync python -m pytest --doctest-modules tests --cov=powercontext --cov-config=pyproject.toml --cov-report=xml:{envtmpdir}/coverage.xml {posargs} + uv run --no-sync ty check